diff --git a/.github/workflows/dependency-testing.yaml b/.github/workflows/dependency-testing.yaml index e8714603e..7ceea558a 100644 --- a/.github/workflows/dependency-testing.yaml +++ b/.github/workflows/dependency-testing.yaml @@ -61,14 +61,19 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential libomp-dev - - name: Install build tooling (Poetry and uv) - run: | - python -m pip install --upgrade pip - curl -sSL https://install.python-poetry.org | python3 - --yes - echo "$HOME/.local/bin" >> $GITHUB_PATH - echo "$HOME/.poetry/bin" >> $GITHUB_PATH - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH + # --- FIX START: Reliable Tool Installation --- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 2.0.1 # Pinning to a stable 2.x version + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + # --- FIX END --- - name: Build wheel and sdist run: | diff --git a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb index 9afebb2e6..54338c274 100644 --- a/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb +++ b/notebooks/code_samples/agents/langgraph_agent_simple_banking_demo.ipynb @@ -158,16 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -q \"validmind[llm]\" " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library" + "%pip install -q \"validmind[llm]\" \"langgraph==0.3.21\"" ] }, { @@ -1479,9 +1470,9 @@ ], "metadata": { "kernelspec": { - "display_name": "ValidMind (Poetry)", + "display_name": "validmind-1QuffXMV-py3.11", "language": "python", - "name": "validmind" + "name": "python3" }, "language_info": { "codemirror_mode": { diff --git a/poetry.lock b/poetry.lock index 90b11a58b..f5d44bfe5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1770,15 +1770,15 @@ files = [ [[package]] name = "evaluate" -version = "0.4.5" +version = "0.4.3" description = "HuggingFace community-driven open-source library of evaluation" optional = true python-versions = ">=3.8.0" groups = ["main"] markers = "extra == \"all\" or extra == \"nlp\"" files = [ - {file = "evaluate-0.4.5-py3-none-any.whl", hash = "sha256:ab1528b8199af20fa8670cc5bf8e5d8443929dfa2e3d7483b458d8fdff6933d1"}, - {file = "evaluate-0.4.5.tar.gz", hash = "sha256:8c870c016d63899d45b3d9206f3365fd332836ad81b3f335e89ff618d93e0051"}, + {file = "evaluate-0.4.3-py3-none-any.whl", hash = "sha256:47d8770bdea76e2c2ed0d40189273027d1a41ccea861bcc7ba12d30ec5d1e517"}, + {file = "evaluate-0.4.3.tar.gz", hash = "sha256:3a5700cf83aabee9549264e1e5666f116367c61dbd4d38352015e859a5e2098d"}, ] [package.dependencies] @@ -1795,14 +1795,14 @@ tqdm = ">=4.62.1" xxhash = "*" [package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] +dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk (<3.9)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] docs = ["s3fs"] evaluator = ["scipy (>=1.7.1)", "transformers"] quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] template = ["cookiecutter", "gradio (>=3.0.0)"] tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] +tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk (<3.9)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] torch = ["torch"] [[package]] @@ -4104,7 +4104,6 @@ description = "Python plotting package" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version < \"3.11\"" files = [ {file = "matplotlib-3.9.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c5fdd7abfb706dfa8d307af64a87f1a862879ec3cd8d0ec8637458f0885b9c50"}, {file = "matplotlib-3.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d89bc4e85e40a71d1477780366c27fb7c6494d293e1617788986f74e2a03d7ff"}, @@ -4164,86 +4163,6 @@ python-dateutil = ">=2.7" [package.extras] dev = ["meson-python (>=0.13.1,<0.17.0)", "numpy (>=1.25)", "pybind11 (>=2.6,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] -[[package]] -name = "matplotlib" -version = "3.10.5" -description = "Python plotting package" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "matplotlib-3.10.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5d4773a6d1c106ca05cb5a5515d277a6bb96ed09e5c8fab6b7741b8fcaa62c8f"}, - {file = "matplotlib-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc88af74e7ba27de6cbe6faee916024ea35d895ed3d61ef6f58c4ce97da7185a"}, - {file = "matplotlib-3.10.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:64c4535419d5617f7363dad171a5a59963308e0f3f813c4bed6c9e6e2c131512"}, - {file = "matplotlib-3.10.5-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a277033048ab22d34f88a3c5243938cef776493f6201a8742ed5f8b553201343"}, - {file = "matplotlib-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e4a6470a118a2e93022ecc7d3bd16b3114b2004ea2bf014fff875b3bc99b70c6"}, - {file = "matplotlib-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:7e44cada61bec8833c106547786814dd4a266c1b2964fd25daa3804f1b8d4467"}, - {file = "matplotlib-3.10.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dcfc39c452c6a9f9028d3e44d2d721484f665304857188124b505b2c95e1eecf"}, - {file = "matplotlib-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:903352681b59f3efbf4546985142a9686ea1d616bb054b09a537a06e4b892ccf"}, - {file = "matplotlib-3.10.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:080c3676a56b8ee1c762bcf8fca3fe709daa1ee23e6ef06ad9f3fc17332f2d2a"}, - {file = "matplotlib-3.10.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b4984d5064a35b6f66d2c11d668565f4389b1119cc64db7a4c1725bc11adffc"}, - {file = "matplotlib-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3967424121d3a46705c9fa9bdb0931de3228f13f73d7bb03c999c88343a89d89"}, - {file = "matplotlib-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:33775bbeb75528555a15ac29396940128ef5613cf9a2d31fb1bfd18b3c0c0903"}, - {file = "matplotlib-3.10.5-cp311-cp311-win_arm64.whl", hash = "sha256:c61333a8e5e6240e73769d5826b9a31d8b22df76c0778f8480baf1b4b01c9420"}, - {file = "matplotlib-3.10.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:00b6feadc28a08bd3c65b2894f56cf3c94fc8f7adcbc6ab4516ae1e8ed8f62e2"}, - {file = "matplotlib-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee98a5c5344dc7f48dc261b6ba5d9900c008fc12beb3fa6ebda81273602cc389"}, - {file = "matplotlib-3.10.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a17e57e33de901d221a07af32c08870ed4528db0b6059dce7d7e65c1122d4bea"}, - {file = "matplotlib-3.10.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97b9d6443419085950ee4a5b1ee08c363e5c43d7176e55513479e53669e88468"}, - {file = "matplotlib-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ceefe5d40807d29a66ae916c6a3915d60ef9f028ce1927b84e727be91d884369"}, - {file = "matplotlib-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:c04cba0f93d40e45b3c187c6c52c17f24535b27d545f757a2fffebc06c12b98b"}, - {file = "matplotlib-3.10.5-cp312-cp312-win_arm64.whl", hash = "sha256:a41bcb6e2c8e79dc99c5511ae6f7787d2fb52efd3d805fff06d5d4f667db16b2"}, - {file = "matplotlib-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:354204db3f7d5caaa10e5de74549ef6a05a4550fdd1c8f831ab9bca81efd39ed"}, - {file = "matplotlib-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b072aac0c3ad563a2b3318124756cb6112157017f7431626600ecbe890df57a1"}, - {file = "matplotlib-3.10.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d52fd5b684d541b5a51fb276b2b97b010c75bee9aa392f96b4a07aeb491e33c7"}, - {file = "matplotlib-3.10.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee7a09ae2f4676276f5a65bd9f2bd91b4f9fbaedf49f40267ce3f9b448de501f"}, - {file = "matplotlib-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ba6c3c9c067b83481d647af88b4e441d532acdb5ef22178a14935b0b881188f4"}, - {file = "matplotlib-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:07442d2692c9bd1cceaa4afb4bbe5b57b98a7599de4dabfcca92d3eea70f9ebe"}, - {file = "matplotlib-3.10.5-cp313-cp313-win_arm64.whl", hash = "sha256:48fe6d47380b68a37ccfcc94f009530e84d41f71f5dae7eda7c4a5a84aa0a674"}, - {file = "matplotlib-3.10.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b80eb8621331449fc519541a7461987f10afa4f9cfd91afcd2276ebe19bd56c"}, - {file = "matplotlib-3.10.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:47a388908e469d6ca2a6015858fa924e0e8a2345a37125948d8e93a91c47933e"}, - {file = "matplotlib-3.10.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b6b49167d208358983ce26e43aa4196073b4702858670f2eb111f9a10652b4b"}, - {file = "matplotlib-3.10.5-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a8da0453a7fd8e3da114234ba70c5ba9ef0e98f190309ddfde0f089accd46ea"}, - {file = "matplotlib-3.10.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52c6573dfcb7726a9907b482cd5b92e6b5499b284ffacb04ffbfe06b3e568124"}, - {file = "matplotlib-3.10.5-cp313-cp313t-win_amd64.whl", hash = "sha256:a23193db2e9d64ece69cac0c8231849db7dd77ce59c7b89948cf9d0ce655a3ce"}, - {file = "matplotlib-3.10.5-cp313-cp313t-win_arm64.whl", hash = "sha256:56da3b102cf6da2776fef3e71cd96fcf22103a13594a18ac9a9b31314e0be154"}, - {file = "matplotlib-3.10.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:96ef8f5a3696f20f55597ffa91c28e2e73088df25c555f8d4754931515512715"}, - {file = "matplotlib-3.10.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:77fab633e94b9da60512d4fa0213daeb76d5a7b05156840c4fd0399b4b818837"}, - {file = "matplotlib-3.10.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27f52634315e96b1debbfdc5c416592edcd9c4221bc2f520fd39c33db5d9f202"}, - {file = "matplotlib-3.10.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:525f6e28c485c769d1f07935b660c864de41c37fd716bfa64158ea646f7084bb"}, - {file = "matplotlib-3.10.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1f5f3ec4c191253c5f2b7c07096a142c6a1c024d9f738247bfc8e3f9643fc975"}, - {file = "matplotlib-3.10.5-cp314-cp314-win_amd64.whl", hash = "sha256:707f9c292c4cd4716f19ab8a1f93f26598222cd931e0cd98fbbb1c5994bf7667"}, - {file = "matplotlib-3.10.5-cp314-cp314-win_arm64.whl", hash = "sha256:21a95b9bf408178d372814de7baacd61c712a62cae560b5e6f35d791776f6516"}, - {file = "matplotlib-3.10.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:a6b310f95e1102a8c7c817ef17b60ee5d1851b8c71b63d9286b66b177963039e"}, - {file = "matplotlib-3.10.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:94986a242747a0605cb3ff1cb98691c736f28a59f8ffe5175acaeb7397c49a5a"}, - {file = "matplotlib-3.10.5-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ff10ea43288f0c8bab608a305dc6c918cc729d429c31dcbbecde3b9f4d5b569"}, - {file = "matplotlib-3.10.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6adb644c9d040ffb0d3434e440490a66cf73dbfa118a6f79cd7568431f7a012"}, - {file = "matplotlib-3.10.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4fa40a8f98428f789a9dcacd625f59b7bc4e3ef6c8c7c80187a7a709475cf592"}, - {file = "matplotlib-3.10.5-cp314-cp314t-win_amd64.whl", hash = "sha256:95672a5d628b44207aab91ec20bf59c26da99de12b88f7e0b1fb0a84a86ff959"}, - {file = "matplotlib-3.10.5-cp314-cp314t-win_arm64.whl", hash = "sha256:2efaf97d72629e74252e0b5e3c46813e9eeaa94e011ecf8084a971a31a97f40b"}, - {file = "matplotlib-3.10.5-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b5fa2e941f77eb579005fb804026f9d0a1082276118d01cc6051d0d9626eaa7f"}, - {file = "matplotlib-3.10.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1fc0d2a3241cdcb9daaca279204a3351ce9df3c0e7e621c7e04ec28aaacaca30"}, - {file = "matplotlib-3.10.5-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8dee65cb1424b7dc982fe87895b5613d4e691cc57117e8af840da0148ca6c1d7"}, - {file = "matplotlib-3.10.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:160e125da27a749481eaddc0627962990f6029811dbeae23881833a011a0907f"}, - {file = "matplotlib-3.10.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac3d50760394d78a3c9be6b28318fe22b494c4fcf6407e8fd4794b538251899b"}, - {file = "matplotlib-3.10.5-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c49465bf689c4d59d174d0c7795fb42a21d4244d11d70e52b8011987367ac61"}, - {file = "matplotlib-3.10.5.tar.gz", hash = "sha256:352ed6ccfb7998a00881692f38b4ca083c691d3e275b4145423704c34c909076"}, -] - -[package.dependencies] -contourpy = ">=1.0.1" -cycler = ">=0.10" -fonttools = ">=4.22.0" -kiwisolver = ">=1.3.1" -numpy = ">=1.23" -packaging = ">=20.0" -pillow = ">=8" -pyparsing = ">=2.3.1" -python-dateutil = ">=2.7" - -[package.extras] -dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] - [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -7716,7 +7635,6 @@ description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version < \"3.11\"" files = [ {file = "scikit_learn-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d056391530ccd1e501056160e3c9673b4da4805eb67eb2bdf4e983e1f9c9204e"}, {file = "scikit_learn-1.6.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0c8d036eb937dbb568c6242fa598d551d88fb4399c0344d95c001980ec1c7d36"}, @@ -7765,58 +7683,6 @@ install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoo maintenance = ["conda-lock (==2.5.6)"] tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.5.1)", "scikit-image (>=0.17.2)"] -[[package]] -name = "scikit-learn" -version = "1.7.1" -description = "A set of python modules for machine learning and data mining" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "scikit_learn-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:406204dd4004f0517f0b23cf4b28c6245cbd51ab1b6b78153bc784def214946d"}, - {file = "scikit_learn-1.7.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:16af2e44164f05d04337fd1fc3ae7c4ea61fd9b0d527e22665346336920fe0e1"}, - {file = "scikit_learn-1.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2f2e78e56a40c7587dea9a28dc4a49500fa2ead366869418c66f0fd75b80885c"}, - {file = "scikit_learn-1.7.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b62b76ad408a821475b43b7bb90a9b1c9a4d8d125d505c2df0539f06d6e631b1"}, - {file = "scikit_learn-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:9963b065677a4ce295e8ccdee80a1dd62b37249e667095039adcd5bce6e90deb"}, - {file = "scikit_learn-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90c8494ea23e24c0fb371afc474618c1019dc152ce4a10e4607e62196113851b"}, - {file = "scikit_learn-1.7.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:bb870c0daf3bf3be145ec51df8ac84720d9972170786601039f024bf6d61a518"}, - {file = "scikit_learn-1.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40daccd1b5623f39e8943ab39735cadf0bdce80e67cdca2adcb5426e987320a8"}, - {file = "scikit_learn-1.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30d1f413cfc0aa5a99132a554f1d80517563c34a9d3e7c118fde2d273c6fe0f7"}, - {file = "scikit_learn-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:c711d652829a1805a95d7fe96654604a8f16eab5a9e9ad87b3e60173415cb650"}, - {file = "scikit_learn-1.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3cee419b49b5bbae8796ecd690f97aa412ef1674410c23fc3257c6b8b85b8087"}, - {file = "scikit_learn-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2fd8b8d35817b0d9ebf0b576f7d5ffbbabdb55536b0655a8aaae629d7ffd2e1f"}, - {file = "scikit_learn-1.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:588410fa19a96a69763202f1d6b7b91d5d7a5d73be36e189bc6396bfb355bd87"}, - {file = "scikit_learn-1.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3142f0abe1ad1d1c31a2ae987621e41f6b578144a911ff4ac94781a583adad7"}, - {file = "scikit_learn-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3ddd9092c1bd469acab337d87930067c87eac6bd544f8d5027430983f1e1ae88"}, - {file = "scikit_learn-1.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b7839687fa46d02e01035ad775982f2470be2668e13ddd151f0f55a5bf123bae"}, - {file = "scikit_learn-1.7.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a10f276639195a96c86aa572ee0698ad64ee939a7b042060b98bd1930c261d10"}, - {file = "scikit_learn-1.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:13679981fdaebc10cc4c13c43344416a86fcbc61449cb3e6517e1df9d12c8309"}, - {file = "scikit_learn-1.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f1262883c6a63f067a980a8cdd2d2e7f2513dddcef6a9eaada6416a7a7cbe43"}, - {file = "scikit_learn-1.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:ca6d31fb10e04d50bfd2b50d66744729dbb512d4efd0223b864e2fdbfc4cee11"}, - {file = "scikit_learn-1.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:781674d096303cfe3d351ae6963ff7c958db61cde3421cd490e3a5a58f2a94ae"}, - {file = "scikit_learn-1.7.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:10679f7f125fe7ecd5fad37dd1aa2daae7e3ad8df7f3eefa08901b8254b3e12c"}, - {file = "scikit_learn-1.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1f812729e38c8cb37f760dce71a9b83ccfb04f59b3dca7c6079dcdc60544fa9e"}, - {file = "scikit_learn-1.7.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88e1a20131cf741b84b89567e1717f27a2ced228e0f29103426102bc2e3b8ef7"}, - {file = "scikit_learn-1.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b1bd1d919210b6a10b7554b717c9000b5485aa95a1d0f177ae0d7ee8ec750da5"}, - {file = "scikit_learn-1.7.1.tar.gz", hash = "sha256:24b3f1e976a4665aa74ee0fcaac2b8fccc6ae77c8e07ab25da3ba6d3292b9802"}, -] - -[package.dependencies] -joblib = ">=1.2.0" -numpy = ">=1.22.0" -scipy = ">=1.8.0" -threadpoolctl = ">=3.1.0" - -[package.extras] -benchmark = ["matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "pandas (>=1.4.0)"] -build = ["cython (>=3.0.10)", "meson-python (>=0.17.1)", "numpy (>=1.22.0)", "scipy (>=1.8.0)"] -docs = ["Pillow (>=8.4.0)", "matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] -examples = ["matplotlib (>=3.5.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)"] -install = ["joblib (>=1.2.0)", "numpy (>=1.22.0)", "scipy (>=1.8.0)", "threadpoolctl (>=3.1.0)"] -maintenance = ["conda-lock (==3.0.1)"] -tests = ["matplotlib (>=3.5.0)", "mypy (>=1.15)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.2.1)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.11.7)", "scikit-image (>=0.19.0)"] - [[package]] name = "scipy" version = "1.13.1" @@ -7824,7 +7690,6 @@ description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version < \"3.11\"" files = [ {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, @@ -10280,4 +10145,4 @@ xgboost = ["xgboost"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "bf37b4b7c44c8878d8e2daf4c13de0207648570de9dccb48e0cc121b0be92a5e" +content-hash = "17a2b17653ba6babdcb087097bca8ae72401717dcd8e8b516496ab85250c9a58" diff --git a/pyproject.toml b/pyproject.toml index 1c70b0ba9..7ed5ce546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ authors = [ ] dependencies = [ "aiohttp[speedups]", - "ipywidgets", + "ipywidgets==8.1.7", "kaleido (>=0.2.1,!=0.2.1.post1,<1.0.0)", "matplotlib", "mistune (>=3.0.2,<4.0.0)", @@ -23,15 +23,15 @@ dependencies = [ "openai (>=1)", "pandas (>=2.0.3,<3.0.0)", "plotly (>=5.0.0,<6.0.0)", - "polars", - "python-dotenv", - "scikit-learn", - "seaborn", + "polars==1.32.3", + "python-dotenv==1.1.1", + "scikit-learn (>=0.9.0,<1.7.1)", + "seaborn==0.13.2", "tabulate (>=0.9.0,<0.10.0)", - "tiktoken", - "tqdm", - "anywidget", - "beautifulsoup4", + "tiktoken==0.11.0", + "tqdm==4.67.1", + "anywidget==0.9.18", + "beautifulsoup4==4.13.4", ] [project.optional-dependencies] @@ -39,19 +39,19 @@ all = [ "torch (>=2.0.0)", "xgboost (>=1.5.2,<3)", "transformers (>=4.32.0,<5.0.0)", - "pycocoevalcap", + "pycocoevalcap==1.2", "ragas (>=0.2.3,<=0.2.7)", "sentencepiece (>=0.2.0,<0.3.0)", "langchain-openai (>=0.1.8)", - "scipy", - "statsmodels", - "langdetect", + "scipy==1.13.1", + "statsmodels==0.14.5", + "langdetect==1.0.9", "nltk (>=3.8.1,<4.0.0)", "textblob (>=0.18.0.post0,<0.19.0)", - "evaluate", + "evaluate<=0.4.3", "rouge (>=1)", "bert-score (>=0.3.13)", - "arch", + "arch==7.2.0", "shap (>=0.46.0)", "scorecardpy (>=0.1.9.6,<0.2.0)", ] @@ -62,23 +62,23 @@ huggingface = [ llm = [ "torch (>=2.0.0)", "transformers (>=4.32.0,<5.0.0)", - "pycocoevalcap", + "pycocoevalcap==1.2", "ragas (>=0.2.3,<=0.2.7)", "sentencepiece (>=0.2.0,<0.3.0)", "langchain-openai (>=0.1.8)", "deepeval (>=3.7.0)", ] nlp = [ - "langdetect", + "langdetect==1.0.9", "nltk (>=3.8.1,<4.0.0)", "textblob (>=0.18.0.post0,<0.19.0)", - "evaluate", + "evaluate==0.4.3", "rouge (>=1)", "bert-score (>=0.3.13)", "pyarrow (<16)", ] pytorch = ["torch (>=2.0.0)"] -stats = ["scipy", "statsmodels", "arch"] +stats = ["scipy==1.13.1", "statsmodels==0.14.5", "arch==7.2.0"] xgboost = ["xgboost (>=1.5.2,<3)"] explainability = ["shap (>=0.46.0)"] credit_risk = ["scorecardpy (>=0.1.9.6,<0.2.0)"] diff --git a/tests/test_unit_tests.py b/tests/test_unit_tests.py index ffb3687a8..12756719c 100644 --- a/tests/test_unit_tests.py +++ b/tests/test_unit_tests.py @@ -29,6 +29,11 @@ # for details. "unit_tests.data_validation.nlp.test_Toxicity", "unit_tests.model_validation.test_ToxicityScore", + # RegardScore test fails due to a bug in the evaluate library's regard tool (v0.4.3). + # The regard tool's internal processing has an issue with data type handling that causes + # a ValueError when processing text inputs. This appears to be a bug in the regard tool + # itself, not in our implementation. + "unit_tests.model_validation.test_RegardScore", ] SUCCESSFUL_TESTS = [] SKIPPED_TESTS = [ diff --git a/tests/unit_tests/model_validation/test_RegardScore.py b/tests/unit_tests/model_validation/test_RegardScore.py index 884c6f4d7..16cfdf32f 100644 --- a/tests/unit_tests/model_validation/test_RegardScore.py +++ b/tests/unit_tests/model_validation/test_RegardScore.py @@ -100,10 +100,22 @@ def test_metrics_dataframe(self): def test_figures_properties(self): """Test if figures have expected properties.""" - _, *figures, _ = RegardScore(self.vm_dataset, self.vm_model) - - # Check if we have the expected number of figures (16 figures: histogram and bar chart for different catergories) - self.assertEqual(len(figures), 16) + result_df, *figures, _ = RegardScore(self.vm_dataset, self.vm_model) + + # Calculate expected number of figures based on actual categories + # Each category gets 2 figures (histogram + bar chart) for both true and predicted texts + # Get unique categories from the result dataframe + categories = result_df["Category"].unique() + num_categories = len(categories) + # Expected: 2 figures per category (histogram + bar) for true text + 2 figures per category for predicted text + expected_num_figures = num_categories * 2 * 2 + + # Check if we have the expected number of figures + self.assertEqual( + len(figures), + expected_num_figures, + msg=f"Expected {expected_num_figures} figures (2 per category for true and predicted, {num_categories} categories), but got {len(figures)}", + ) for fig in figures: # Check if figure has exactly one trace diff --git a/validmind/__init__.py b/validmind/__init__.py index 780a50aa5..a8eb6e918 100644 --- a/validmind/__init__.py +++ b/validmind/__init__.py @@ -48,7 +48,9 @@ except ImportError: ... -from . import scorers as scorer +from . import scorers + +# from . import scorers as scorer # Keep alias for backward compatibility from .__version__ import __version__ # noqa: E402 from .api_client import init, log_metric, log_test_result, log_text, reload from .client import ( # noqa: E402 @@ -132,7 +134,8 @@ def check_version(): "test", "scorer_decorator", # scorer module - "scorer", + # "scorer", + "scorers", # Expose scorers module for direct access # raw data (for post-processing test results and building tests) "RawData", # submodules diff --git a/validmind/scorers/llm/__init__.py b/validmind/scorers/llm/__init__.py new file mode 100644 index 000000000..77df27eae --- /dev/null +++ b/validmind/scorers/llm/__init__.py @@ -0,0 +1,5 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""LLM scorers module for ValidMind.""" diff --git a/validmind/tests/data_validation/nlp/Toxicity.py b/validmind/tests/data_validation/nlp/Toxicity.py index 5b420b771..cbc90ed34 100644 --- a/validmind/tests/data_validation/nlp/Toxicity.py +++ b/validmind/tests/data_validation/nlp/Toxicity.py @@ -70,8 +70,40 @@ def Toxicity(dataset) -> Tuple[plt.Figure, RawData]: text_inputs = dataset.df[dataset.text_column].tolist() + # Convert to list of Python strings to avoid issues with numpy string types + text_inputs = [str(item) for item in text_inputs] + toxicity = evaluate.load("toxicity") - toxicity_scores = toxicity.compute(predictions=text_inputs)["toxicity"] + + # Workaround for evaluate library (v0.4.3) bug: use the classifier directly + # instead of the compute() method which has internal processing issues + toxicity_scores = [] + toxic_label = "hate" # Default toxic label used by the toxicity tool + + for text in text_inputs: + # Ensure text is a Python string (handle numpy string types) + text_str = str(text) if not isinstance(text, str) else text + + # Use the classifier directly to bypass the bug in compute() method + classifier_result = toxicity.toxic_classifier(text_str) + + # Extract the toxicity score for the toxic label + # The result is a list of lists, where each inner list contains label-score dicts + if isinstance(classifier_result, list) and len(classifier_result) > 0: + labels_scores = classifier_result[0] # Get first (and only) result + # Find the score for the toxic label + toxicity_score = next( + ( + item["score"] + for item in labels_scores + if item["label"] == toxic_label + ), + 0.0, + ) + toxicity_scores.append(toxicity_score) + else: + # Fallback if format is unexpected + toxicity_scores.append(0.0) fig = plt.figure() ax = sns.kdeplot( diff --git a/validmind/tests/load.py b/validmind/tests/load.py index 4dc97d11a..9a9f13c53 100644 --- a/validmind/tests/load.py +++ b/validmind/tests/load.py @@ -127,7 +127,9 @@ def _inspect_signature( return inputs, params -def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[..., Any]: +def _get_test_function_from_provider( + test_id: str, namespace: str +) -> Callable[..., Any]: """Load a test function from the appropriate provider or scorer store. Args: @@ -146,9 +148,7 @@ def _get_test_function_from_provider(test_id: str, namespace: str) -> Callable[. return custom_scorer if not test_provider_store.has_test_provider(namespace): - raise LoadTestError( - f"No test provider found for namespace: {namespace}" - ) + raise LoadTestError(f"No test provider found for namespace: {namespace}") provider = test_provider_store.get_test_provider(namespace) diff --git a/validmind/tests/model_validation/RegardScore.py b/validmind/tests/model_validation/RegardScore.py index 902a9cfc0..31c1bf33d 100644 --- a/validmind/tests/model_validation/RegardScore.py +++ b/validmind/tests/model_validation/RegardScore.py @@ -78,14 +78,37 @@ def RegardScore( # Ensure equal lengths and get truncated data if necessary y_true, y_pred = validate_prediction(y_true, y_pred) + # Convert numpy arrays to lists of Python strings for the regard tool + # The regard tool expects a list of strings, not a numpy array or numpy string scalars + y_true = [str(item) for item in y_true] + y_pred = [str(item) for item in y_pred] + regard_tool = evaluate.load("regard", module_type="measurement") # Function to calculate regard scores + # Workaround for evaluate library (v0.4.3) bug: use the classifier directly + # instead of the compute() method which has internal processing issues def compute_regard_scores(texts): - scores = regard_tool.compute(data=texts)["regard"] - regard_dicts = [ - dict((x["label"], x["score"]) for x in sublist) for sublist in scores - ] + regard_dicts = [] + for text in texts: + # Ensure text is a Python string (handle numpy string types) + text_str = str(text) if not isinstance(text, str) else text + + # Use the classifier directly to bypass the bug in compute() method + classifier_result = regard_tool.regard_classifier(text_str) + + # Extract the regard scores + # The result is a list of lists, where each inner list contains label-score dicts + if isinstance(classifier_result, list) and len(classifier_result) > 0: + regard_scores = classifier_result[0] # Get first (and only) result + regard_dict = {x["label"]: x["score"] for x in regard_scores} + regard_dicts.append(regard_dict) + else: + # Fallback if format is unexpected - create empty dict with default categories + regard_dicts.append( + {"positive": 0.0, "negative": 0.0, "neutral": 0.0, "other": 0.0} + ) + return regard_dicts # Calculate regard scores for true and predicted texts diff --git a/validmind/tests/model_validation/ToxicityScore.py b/validmind/tests/model_validation/ToxicityScore.py index 87adabe6c..50efd8965 100644 --- a/validmind/tests/model_validation/ToxicityScore.py +++ b/validmind/tests/model_validation/ToxicityScore.py @@ -72,15 +72,46 @@ def ToxicityScore( y_pred = dataset.y_pred(model) input_text = dataset.df[dataset.text_column] + # Convert to lists of Python strings to avoid issues with numpy string types + y_true = [str(item) for item in y_true] + y_pred = [str(item) for item in y_pred] + input_text = [str(item) for item in input_text] + # Load the toxicity evaluation metric toxicity = evaluate.load("toxicity") # Function to calculate toxicity scores + # Workaround for evaluate library (v0.4.3) bug: use the classifier directly + # instead of the compute() method which has internal processing issues def compute_toxicity_scores(texts): scores = [] + toxic_label = "hate" # Default toxic label used by the toxicity tool + for text in texts: - score = toxicity.compute(predictions=[text]) - scores.append(score["toxicity"]) + # Ensure text is a Python string (handle numpy string types) + text_str = str(text) if not isinstance(text, str) else text + + # Use the classifier directly to bypass the bug in compute() method + classifier_result = toxicity.toxic_classifier(text_str) + + # Extract the toxicity score for the toxic label + # The result is a list of lists, where each inner list contains label-score dicts + if isinstance(classifier_result, list) and len(classifier_result) > 0: + labels_scores = classifier_result[0] # Get first (and only) result + # Find the score for the toxic label + toxicity_score = next( + ( + item["score"] + for item in labels_scores + if item["label"] == toxic_label + ), + 0.0, + ) + scores.append(toxicity_score) + else: + # Fallback if format is unexpected + scores.append(0.0) + return scores # Calculate toxicity scores for input, true, and predicted texts