diff --git a/ST/.DS_Store b/ST/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c69853d757f12c2deebfdba76312b723b6a16a43 Binary files /dev/null and b/ST/.DS_Store differ diff --git a/ST/evaluate/.DS_Store b/ST/evaluate/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..dffb22ae51c007d806d65b4e04142b92226cf4e2 Binary files /dev/null and b/ST/evaluate/.DS_Store differ diff --git a/ST/evaluate/.github/hub/push_evaluations_to_hub.py b/ST/evaluate/.github/hub/push_evaluations_to_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..73bde44cebce2c2574357516d4e070dee44d8f8e --- /dev/null +++ b/ST/evaluate/.github/hub/push_evaluations_to_hub.py @@ -0,0 +1,118 @@ +from pathlib import Path +from huggingface_hub import create_repo, Repository +import tempfile +import subprocess +import os +import shutil +import logging +import re +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +GIT_UP_TO_DATE = "On branch main\nYour branch is up to date with 'origin/main'.\ +\n\nnothing to commit, working tree clean\n" + +COMMIT_PLACEHOLDER = "{COMMIT_PLACEHOLDER}" + +def get_git_tag(lib_path, commit_hash): + # check if commit has a tag, see: https://stackoverflow.com/questions/1474115/how-to-find-the-tag-associated-with-a-given-git-commit + command = f"git describe --exact-match {commit_hash}" + output = subprocess.run(command.split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf-8", + cwd=lib_path, + env=os.environ.copy(), + ) + tag = output.stdout.strip() + if re.match(r"v\d*\.\d*\.\d*", tag) is not None: + return tag + else: + return None + + +def copy_recursive(source_base_path, target_base_path): + """Copy directory recursively and overwrite existing files.""" + for item in source_base_path.iterdir(): + target_path = target_base_path / item.name + if item.is_dir(): + target_path.mkdir(exist_ok=True) + copy_recursive(item, target_path) + else: + shutil.copy(item, target_path) + +def update_evaluate_dependency(requirements_path, commit_hash): + """Updates the evaluate requirement with the latest commit.""" + with open(requirements_path, "r") as f: + file_content = f.read() + file_content = file_content.replace(COMMIT_PLACEHOLDER, commit_hash) + with open(requirements_path, "w") as f: + f.write(file_content) + +def push_module_to_hub(module_path, type, token, commit_hash, tag=None): + module_name = module_path.stem + org = f"evaluate-{type}" + + repo_url = create_repo(org + "/" + module_name, repo_type="space", space_sdk="gradio", exist_ok=True, token=token) + repo_path = Path(tempfile.mkdtemp()) + + scheme = urlparse(repo_url).scheme + repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@") + clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url) + + try: + subprocess.run( + f"git clone {repo_url}".split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + check=True, + encoding="utf-8", + cwd=repo_path, + env=os.environ.copy(), + ) + except OSError: + # make sure we don't accidentally expose token + raise OSError(f"Could not clone from '{clean_repo_url}'") + + repo = Repository(local_dir=repo_path / module_name, use_auth_token=token) + + copy_recursive(module_path, repo_path / module_name) + update_evaluate_dependency(repo_path / module_name / "requirements.txt", commit_hash) + + repo.git_add() + try: + repo.git_commit(f"Update Space (evaluate main: {commit_hash[:8]})") + repo.git_push() + logger.info(f"Module '{module_name}' pushed to the hub") + except OSError as error: + if str(error) == GIT_UP_TO_DATE: + logger.info(f"Module '{module_name}' is already up to date.") + else: + raise error + + if tag is not None: + repo.add_tag(tag, message="add evaluate tag", remote="origin") + + shutil.rmtree(repo_path) + + +if __name__ == "__main__": + evaluation_paths = ["metrics", "comparisons", "measurements"] + evaluation_types = ["metric", "comparison", "measurement"] + + token = os.getenv("HF_TOKEN") + evaluate_lib_path = Path(os.getenv("EVALUATE_LIB_PATH")) + commit_hash = os.getenv("GIT_HASH") + git_tag = get_git_tag(evaluate_lib_path, commit_hash) + if git_tag is not None: + logger.info(f"Found tag: {git_tag}.") + + for type, dir in zip(evaluation_types, evaluation_paths): + if (evaluate_lib_path/dir).exists(): + for module_path in (evaluate_lib_path/dir).iterdir(): + if module_path.is_dir(): + logger.info(f"Updating: module {module_path.name}.") + push_module_to_hub(module_path, type, token, commit_hash, tag=git_tag) + else: + logger.warning(f"No folder {str(evaluate_lib_path/dir)} for {type} found.") diff --git a/ST/evaluate/.github/hub/requirements.txt b/ST/evaluate/.github/hub/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b2642b32880c428fa0e39156712a15479b556995 --- /dev/null +++ b/ST/evaluate/.github/hub/requirements.txt @@ -0,0 +1 @@ +huggingface_hub \ No newline at end of file diff --git a/ST/evaluate/.github/workflows/build_documentation.yml b/ST/evaluate/.github/workflows/build_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..19d826a213f6ccb5ab5bf8d4633a9397c33c4334 --- /dev/null +++ b/ST/evaluate/.github/workflows/build_documentation.yml @@ -0,0 +1,17 @@ +name: Build documentation + +on: + push: + branches: + - main + - doc-builder* + - v*-release + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: evaluate + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} \ No newline at end of file diff --git a/ST/evaluate/.github/workflows/build_pr_documentation.yml b/ST/evaluate/.github/workflows/build_pr_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..42aea774be61d97be92e6a50ea020389e28bf743 --- /dev/null +++ b/ST/evaluate/.github/workflows/build_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: evaluate \ No newline at end of file diff --git a/ST/evaluate/.github/workflows/ci.yml b/ST/evaluate/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..21dde7a3befdc680a4824a2cf096478b5ddba498 --- /dev/null +++ b/ST/evaluate/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: CI + +on: + pull_request: + branches: + - main + push: + branches: + - main + - ci-* + +env: + HF_ALLOW_CODE_EVAL: 1 + +jobs: + + check_code_quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.7" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[quality] + - name: Check quality + run: | + black --check --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort --check-only tests src metrics comparisons measurements + flake8 tests src metrics + + test: + needs: check_code_quality + strategy: + matrix: + test: ['unit', 'parity'] + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python 3.7 + uses: actions/setup-python@v4 + with: + python-version: "3.7" + - name: Upgrade pip + run: python -m pip install --upgrade pip + - name: Install dependencies + run: | + pip install .[tests] + pip install -r additional-tests-requirements.txt --no-deps + - name: Test with pytest + if: ${{ matrix.test == 'unit' }} + run: | + python -m pytest -n 2 --dist loadfile -sv ./tests/ --ignore=./tests/test_trainer_evaluator_parity.py + - name: Integration test with transformers + if: ${{ matrix.test == 'parity' }} + run: | + python -m pytest -n 2 --dist loadfile -sv ./tests/test_trainer_evaluator_parity.py diff --git a/ST/evaluate/.github/workflows/delete_doc_comment.yml b/ST/evaluate/.github/workflows/delete_doc_comment.yml new file mode 100644 index 0000000000000000000000000000000000000000..a835cc6f8f6b5b8e54d6b5e23d0ac2016dfc020c --- /dev/null +++ b/ST/evaluate/.github/workflows/delete_doc_comment.yml @@ -0,0 +1,13 @@ +name: Delete dev documentation + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main + with: + pr_number: ${{ github.event.number }} + package: evaluate \ No newline at end of file diff --git a/ST/evaluate/.github/workflows/python-release.yml b/ST/evaluate/.github/workflows/python-release.yml new file mode 100644 index 0000000000000000000000000000000000000000..d680802aa311832714d15c86b8073812da2dc948 --- /dev/null +++ b/ST/evaluate/.github/workflows/python-release.yml @@ -0,0 +1,31 @@ +name: Python release + +on: + push: + tags: + - v* + +env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} + +jobs: + python_release: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + pip install --upgrade pip + pip install setuptools wheel + - run: python setup.py sdist bdist_wheel + + - run: | + pip install twine + - name: Upload to PyPi + run: | + twine upload dist/* -u __token__ -p "$PYPI_TOKEN" diff --git a/ST/evaluate/.github/workflows/update_spaces.yml b/ST/evaluate/.github/workflows/update_spaces.yml new file mode 100644 index 0000000000000000000000000000000000000000..7ecfe1860340b883bcebd5ba28d3b38a44d9199b --- /dev/null +++ b/ST/evaluate/.github/workflows/update_spaces.yml @@ -0,0 +1,36 @@ +name: Update Hub repositories + +on: + push: + branches: + - main + +jobs: + update-hub-repositories: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Set up default Git config + run: | + git config --global user.name evaluate-bot + git config --global user.email leandro@huggingface.co + - name: Install dependencies + working-directory: ./.github/hub + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Update Hub repositories + working-directory: ./.github/hub + run: | + export HF_TOKEN=${{ secrets.HF_HUB_TOKEN }} + export EVALUATE_LIB_PATH=$GITHUB_WORKSPACE + export GIT_HASH=$GITHUB_SHA + export GIT_LFS_SKIP_SMUDGE=1 + python push_evaluations_to_hub.py \ No newline at end of file diff --git a/ST/evaluate/.gitignore b/ST/evaluate/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b4b4d526dd988db2373d9c9cdaa24a894a581281 --- /dev/null +++ b/ST/evaluate/.gitignore @@ -0,0 +1,64 @@ +# Locked files +*.lock +!dvc.lock + +# Extracted dummy data +datasets/**/dummy_data-zip-extracted/ + +# Compiled python modules. +*.pyc + +# Byte-compiled +_pycache__/ +.cache/ + +# Python egg metadata, regenerated from source files by setuptools. +*.egg-info +.eggs/ + +# PyPI distribution artifacts. +build/ +dist/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# pyenv +.python-version + +# Tests +.pytest_cache/ + +# Other +*.DS_Store + +# PyCharm/vscode +.idea +.vscode + +# keep only the empty datasets and metrics directory with it's __init__.py file +/src/*/datasets/* +!/src/*/datasets/__init__.py + +/src/*/metrics/* +!/src/*/metrics/__init__.py + +# Vim +.*.swp + +# playground +/playground + +# Sphinx documentation +docs/_build/ +docs/source/_build/ + +# Benchmark results +report.json +report.md \ No newline at end of file diff --git a/ST/evaluate/AUTHORS b/ST/evaluate/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..7092efd949bb407beb485f25d6c1847804a79170 --- /dev/null +++ b/ST/evaluate/AUTHORS @@ -0,0 +1,8 @@ +# This is the list of HuggingFace Datasets authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. + +Google Inc. +HuggingFace Inc. diff --git a/ST/evaluate/CODE_OF_CONDUCT.md b/ST/evaluate/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..450a63cc303b2eceb983656a6201e7702b8a56d0 --- /dev/null +++ b/ST/evaluate/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +feedback@huggingface.co. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available +at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/ST/evaluate/CONTRIBUTING.md b/ST/evaluate/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..c0dac60b53feff8707e11082c11e76d93d95221a --- /dev/null +++ b/ST/evaluate/CONTRIBUTING.md @@ -0,0 +1,277 @@ + + +# How to contribute to Evaluate + +Everyone is welcome to contribute, and we value everybody's contribution. Code +is not the only way to help the community. Answering questions, helping +others, reaching out and improving the documentations are immensely valuable to +the community. + +It also helps us if you spread the word: reference the library from blog posts +on the awesome projects it made possible, shout out on Twitter every time it has +helped you, or simply star the repo to say "thank you". + +Whichever way you choose to contribute, please be mindful to respect our +[code of conduct](https://github.com/huggingface/evaluate/blob/main/CODE_OF_CONDUCT.md). + +## You can contribute in so many ways! + +There are four ways you can contribute to `evaluate`: +* Fixing outstanding issues with the existing code; +* Implementing new evaluators and metrics; +* Contributing to the examples and documentation; +* Submitting issues related to bugs or desired new features. + +Open issues are tracked directly on the repository [here](https://github.com/huggingface/evaluate/issues). + +If you would like to work on any of the open issues: +* Make sure it is not already assigned to someone else. The assignee (if any) is on the top right column of the Issue page. If it's not taken, self-assign it. +* Work on your self-assigned issue and create a Pull Request! + +## Submitting a new issue or feature request + +Following these guidelines when submitting an issue or a feature +request will make it easier for us to come back to you quickly and with good +feedback. + +### Do you want to implement a new metric? + +All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). Evaluation modules can be either **community** or **canonical**. + +* **Canonical** metrics are well-established metrics which already broadly adopted. +* **Community** metrics are new or custom metrics. It is simple to add a new community metric to use with `evaluate`. Please see our guide to adding a new evaluation metric [here](https://huggingface.co/docs/evaluate/creating_and_sharing)! + +The only functional difference is that canonical metrics are integrated into the `evaluate` library directly and do not require a namespace when being loaded. + +We encourage contributors to share new evaluation modules they contribute broadly! If they become widely adopted then they will be integrated into the core `evaluate` library as a canonical module. + +### Do you want to request a new feature (that is not a metric)? + +We would appreciate it if your feature request addresses the following points: + +1. Motivation first: + * Is it related to a problem/frustration with the library? If so, please explain + why. Providing a code snippet that demonstrates the problem is best. + * Is it related to something you would need for a project? We'd love to hear + about it! + * Is it something you worked on and think could benefit the community? + Awesome! Tell us what problem it solved for you. +2. Write a *full paragraph* describing the feature; +3. Provide a **code snippet** that demonstrates its future use; +4. In case this is related to a paper, please attach a link; +5. Attach any additional information (drawings, screenshots, etc.) you think may help. + +### Did you find a bug? + +Thank you for reporting an issue. If the bug is related to a community metric, please open an issue or pull request directly on the repository of the metric on the Hugging Face Hub. + +If the bug is related to the `evaluate` library and not a community metric, we would really appreciate it if you could **make sure the bug was not already reported** (use the search bar on Github under Issues). If it's not already logged, please open an issue with these details: + +* Include your **OS type and version**, the versions of **Python**, **PyTorch** and + **Tensorflow** when applicable; +* A short, self-contained, code snippet that allows us to reproduce the bug in + less than 30s; +* Provide the *full* traceback if an exception is raised. + +## Start contributing! (Pull Requests) + +Before writing code, we strongly advise you to search through the existing PRs or +issues to make sure that nobody is already working on the same thing. If you are +unsure, it is always a good idea to open an issue to get some feedback. + +1. Fork the [repository](https://github.com/huggingface/evaluate) by + clicking on the 'Fork' button on the repository's page. This creates a copy of the code + under your GitHub user account. + +2. Clone your fork to your local disk, and add the base repository as a remote: + + ```bash + $ git clone git@github.com:/evaluate.git + $ cd evaluate + $ git remote add upstream https://github.com/huggingface/evaluate.git + ``` + +3. Create a new branch to hold your development changes: + + ```bash + $ git checkout -b a-descriptive-name-for-my-changes + ``` + + **Do not** work on the `main` branch. + +4. Set up a development environment by running the following command in a virtual environment: + + ```bash + $ pip install -e ".[dev]" + ``` + +5. Develop the features on your branch. + + As you work on the features, you should make sure that the test suite + passes. You should run the tests impacted by your changes like this: + + ```bash + $ pytest tests/.py + ``` + + To run a specific test, for example the `test_model_init` test in test_evaluator.py, + + ```bash + python -m pytest ./tests/test_evaluator.py::TestQuestionAnsweringEvaluator::test_model_init + ``` + + You can also run the full suite with the following command: + + ```bash + $ python -m pytest ./tests/ + ``` + + 🤗 Evaluate relies on `black` and `isort` to format its source code + consistently. After you make changes, apply automatic style corrections and code verifications + that can't be automated in one go with: + + ```bash + $ make fixup + ``` + + This target is also optimized to only work with files modified by the PR you're working on. + + If you prefer to run the checks one after the other, the following command apply the + style corrections: + + ```bash + $ make style + ``` + + 🤗 Evaluate also uses `flake8` and a few custom scripts to check for coding mistakes. Quality + control runs in CI, however you can also run the same checks with: + + ```bash + $ make quality + ``` + + If you're modifying documents under `docs/source`, make sure to validate that + they can still be built. This check also runs in CI. To run a local check + make sure you have installed the documentation builder requirements. First you will need to clone the + repository containing our tools to build the documentation: + + ```bash + $ pip install git+https://github.com/huggingface/doc-builder + ``` + + Then, make sure you have all the dependencies to be able to build the doc with: + + ```bash + $ pip install ".[docs]" + ``` + + Finally, run the following command from the root of the repository: + + ```bash + $ doc-builder build evaluate docs/source/ --build_dir ~/tmp/test-build + ``` + + This will build the documentation in the `~/tmp/test-build` folder where you can inspect the generated + Markdown files with your favorite editor. You won't be able to see the final rendering on the website + before your PR is merged, we are actively working on adding a tool for this. + + Once you're happy with your changes, add changed files using `git add` and + make a commit with `git commit` to record your changes locally: + + ```bash + $ git add modified_file.py + $ git commit + ``` + + Please write [good commit + messages](https://chris.beams.io/posts/git-commit/). + + It is a good idea to sync your copy of the code with the original + repository regularly. This way you can quickly account for changes: + + ```bash + $ git fetch upstream + $ git rebase upstream/main + ``` + + Push the changes to your account using: + + ```bash + $ git push -u origin a-descriptive-name-for-my-changes + ``` + +6. Once you are satisfied, go to the webpage of your fork on GitHub. Click on 'Pull request' to send your changes + to the project maintainers for review. + +7. It's ok if maintainers ask you for changes. It happens to core contributors + too! So everyone can see the changes in the Pull request, work in your local + branch and push the changes to your fork. They will automatically appear in + the pull request. + + +### Checklist + +1. The title of your pull request should be a summary of its contribution; +2. If your pull request addresses an issue, please mention the issue number in + the pull request description to make sure they are linked (and people + consulting the issue know you are working on it); +3. To indicate a work in progress please prefix the title with `[WIP]`. These + are useful to avoid duplicated work, and to differentiate it from PRs ready + to be merged; +4. Make sure existing tests pass; +5. Add high-coverage tests. No quality testing = no merge. +6. All public methods must have informative docstrings that work nicely with sphinx. +7. Due to the rapidly growing repository, it is important to make sure that no files that would significantly weigh down the repository are added. This includes images, videos and other non-text files. We prefer to leverage a hf.co hosted `dataset` like + the ones hosted on [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) in which to place these files and reference + them by URL. + + +### Style guide + +For documentation strings, 🤗 Evaluate follows the [google style](https://google.github.io/styleguide/pyguide.html). +Check our [documentation writing guide](https://github.com/huggingface/transformers/tree/main/docs#writing-documentation---specification) +for more information. + +**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md).** + +### Develop on Windows + +On Windows, you need to configure git to transform Windows `CRLF` line endings to Linux `LF` line endings: + +`git config core.autocrlf input` + +One way one can run the make command on Window is to pass by MSYS2: + +1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64 +2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu) +3. Run in the shell: `pacman -Syu` and install make with `pacman -S make` +4. Add `C:\msys64\usr\bin` to your PATH environment variable. + +You can now use `make` from any terminal (Powershell, cmd.exe, etc) 🎉 + +### Syncing forked main with upstream (HuggingFace) main + +To avoid pinging the upstream repository which adds reference notes to each upstream PR and sends unnecessary notifications to the developers involved in these PRs, +when syncing the main branch of a forked repository, please, follow these steps: +1. When possible, avoid syncing with the upstream using a branch and PR on the forked repository. Instead, merge directly into the forked main. +2. If a PR is absolutely necessary, use the following steps after checking out your branch: +``` +$ git checkout -b your-branch-for-syncing +$ git pull --squash --no-commit upstream main +$ git commit -m '' +$ git push --set-upstream origin your-branch-for-syncing +``` diff --git a/ST/evaluate/LICENSE b/ST/evaluate/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/ST/evaluate/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ST/evaluate/Makefile b/ST/evaluate/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f8ed3611fc9393380b665d7404368a92baab7ad8 --- /dev/null +++ b/ST/evaluate/Makefile @@ -0,0 +1,19 @@ +.PHONY: quality style test + +# Check that source code meets quality standards + +quality: + black --check --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort --check-only tests src metrics measurements + flake8 tests src metrics + +# Format source code automatically + +style: + black --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort tests src metrics measurements + +# Run tests for the library + +test: + python -m pytest -n auto --dist=loadfile -s -v ./tests/ diff --git a/ST/evaluate/README.md b/ST/evaluate/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21747f67551a78f21493903b5fecc8bec37ee329 --- /dev/null +++ b/ST/evaluate/README.md @@ -0,0 +1,78 @@ +

+
+ +
+

+ +

+ + Build + + + GitHub + + + Documentation + + + GitHub release + + + Contributor Covenant + +

+ +🤗 Evaluate is a library that makes evaluating and comparing models and reporting their performance easier and more standardized. + +It currently contains: + +- **implementations of dozens of popular metrics**: the existing metrics cover a variety of tasks spanning from NLP to Computer Vision, and include dataset-specific metrics for datasets. With a simple command like `accuracy = load("accuracy")`, get any of these metrics ready to use for evaluating a ML model in any framework (Numpy/Pandas/PyTorch/TensorFlow/JAX). +- **comparisons and measurements**: comparisons are used to measure the difference between models and measurements are tools to evaluate datasets. +- **an easy way of adding new evaluation modules to the 🤗 Hub**: you can create new evaluation modules and push them to a dedicated Space in the 🤗 Hub with `evaluate-cli create [metric name]`, which allows you to see easily compare different metrics and their outputs for the same sets of references and predictions. + +[🎓 **Documentation**](https://huggingface.co/docs/evaluate/) + +🔎 **Find a [metric](https://huggingface.co/evaluate-metric), [comparison](https://huggingface.co/evaluate-comparison), [measurement](https://huggingface.co/evaluate-measurement) on the Hub** + +[🌟 **Add a new evaluation module**](https://huggingface.co/docs/evaluate/) + +🤗 Evaluate also has lots of useful features like: + +- **Type checking**: the input types are checked to make sure that you are using the right input formats for each metric +- **Metric cards**: each metrics comes with a card that describes the values, limitations and their ranges, as well as providing examples of their usage and usefulness. +- **Community metrics:** Metrics live on the Hugging Face Hub and you can easily add your own metrics for your project or to collaborate with others. + + +# Installation + +## With pip + +🤗 Evaluate can be installed from PyPi and has to be installed in a virtual environment (venv or conda for instance) + +```bash +pip install evaluate +``` + +# Usage + +🤗 Evaluate's main methods are: + +- `evaluate.list_evaluation_modules()` to list the available metrics, comparisons and measurements +- `evaluate.load(module_name, **kwargs)` to instantiate an evaluation module +- `results = module.compute(*kwargs)` to compute the result of an evaluation module + +# Adding a new evaluation module + +First install the necessary dependencies to create a new metric with the following command: +```bash +pip install evaluate[template] +``` +Then you can get started with the following command which will create a new folder for your metric and display the necessary steps: +```bash +evaluate-cli create "Awesome Metric" +``` +See this [step-by-step guide](https://huggingface.co/docs/evaluate/creating_and_sharing) in the documentation for detailed instructions. + +## Credits + +Thanks to [@marella](https://github.com/marella) for letting us use the `evaluate` namespace on PyPi previously used by his [library](https://github.com/marella/evaluate). diff --git a/ST/evaluate/additional-tests-requirements.txt b/ST/evaluate/additional-tests-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53b402e563499116a50c2f50222b666b76c3a943 --- /dev/null +++ b/ST/evaluate/additional-tests-requirements.txt @@ -0,0 +1,6 @@ +unbabel-comet>=1.0.0;python_version>'3.6' +git+https://github.com/google-research/bleurt.git +git+https://github.com/ns-moosavi/coval.git +git+https://github.com/hendrycks/math.git +git+https://github.com/google-research/rl-reliability-metrics +gin-config \ No newline at end of file diff --git a/ST/evaluate/comparisons/exact_match/README.md b/ST/evaluate/comparisons/exact_match/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7370e289d29c603a33efddaf63ea4b140a1ae144 --- /dev/null +++ b/ST/evaluate/comparisons/exact_match/README.md @@ -0,0 +1,61 @@ +--- +title: Exact Match +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + Returns the rate at which the predictions of one model exactly match those of another model. +--- + + +# Comparison Card for Exact Match + +## Comparison description + + Given two model predictions the exact match score is 1 if they are the exact same, and is 0 otherwise. The overall exact match score is the average. + +- **Example 1**: The exact match score if prediction 1.0 is [0, 1] is 0, given prediction 2 is [0, 1]. +- **Example 2**: The exact match score if prediction 0.0 is [0, 1] is 0, given prediction 2 is [1, 0]. +- **Example 3**: The exact match score if prediction 0.5 is [0, 1] is 0, given prediction 2 is [1, 1]. + +## How to use + +At minimum, this metric takes as input predictions and references: +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 1, 1], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 0.66} +``` + +## Output values + +Returns a float between 0.0 and 1.0 inclusive. + +## Examples + +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 0, 0], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 1.0} +``` + +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 1, 1], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 0.66} +``` + + +## Limitations and bias + +## Citations diff --git a/ST/evaluate/comparisons/exact_match/app.py b/ST/evaluate/comparisons/exact_match/app.py new file mode 100644 index 0000000000000000000000000000000000000000..9efaa644a2e41fe749ddc3849a9b162b48c29cec --- /dev/null +++ b/ST/evaluate/comparisons/exact_match/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("exact_match", module_type="comparison") +launch_gradio_widget(module) diff --git a/ST/evaluate/comparisons/exact_match/exact_match.py b/ST/evaluate/comparisons/exact_match/exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..08c300137c6cb6c4c27a86262f2ffd306e77db15 --- /dev/null +++ b/ST/evaluate/comparisons/exact_match/exact_match.py @@ -0,0 +1,65 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exact match test for model comparison.""" + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Returns the rate at which the predictions of one model exactly match those of another model. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `int`): Predicted labels for model 1. + predictions2 (`list` of `int`): Predicted labels for model 2. + +Returns: + exact_match (`float`): Dictionary containing exact_match rate. Possible values are between 0.0 and 1.0, inclusive. + +Examples: + >>> exact_match = evaluate.load("exact_match", module_type="comparison") + >>> results = exact_match.compute(predictions1=[1, 1, 1], predictions2=[1, 1, 1]) + >>> print(results) + {'exact_match': 1.0} +""" + + +_CITATION = """ +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ExactMatch(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("int64"), + "predictions2": datasets.Value("int64"), + } + ), + ) + + def _compute(self, predictions1, predictions2): + score_list = [p1 == p2 for p1, p2 in zip(predictions1, predictions2)] + return {"exact_match": np.mean(score_list)} diff --git a/ST/evaluate/comparisons/exact_match/requirements.txt b/ST/evaluate/comparisons/exact_match/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/ST/evaluate/comparisons/exact_match/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/ST/evaluate/comparisons/mcnemar/README.md b/ST/evaluate/comparisons/mcnemar/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ceaaee211a3abb180cd0f4e3eb82459523c8a75 --- /dev/null +++ b/ST/evaluate/comparisons/mcnemar/README.md @@ -0,0 +1,86 @@ +--- +title: McNemar +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + McNemar's test is a diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: + McNemar = (SE - SP)**2 / SE + SP + Where: + SE: Sensitivity (Test 1 positive; Test 2 negative) + SP: Specificity (Test 1 negative; Test 2 positive) +--- + + +# Comparison Card for McNemar + +## Comparison description + +McNemar's test is a non-parametric diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: + +McNemar = (SE - SP)**2 / SE + SP + +Where: +* SE: Sensitivity (Test 1 positive; Test 2 negative) +* SP: Specificity (Test 1 negative; Test 2 positive) + +In other words, SE and SP are the diagonal elements of the contingency table for the classifier predictions (`predictions1` and `predictions2`) with respect to the ground truth `references`. + +## How to use + +The McNemar comparison calculates the proportions of responses that exhibit disagreement between two classifiers. It is used to analyze paired nominal data. + +## Inputs + +Its arguments are: + +`predictions1`: a list of predictions from the first model. + +`predictions2`: a list of predictions from the second model. + +`references`: a list of the ground truth reference labels. + +## Output values + +The McNemar comparison outputs two things: + +`stat`: The McNemar statistic. + +`p`: The p value. + +## Examples + +Example comparison: + +```python +mcnemar = evaluate.load("mcnemar") +results = mcnemar.compute(references=[1, 0, 1], predictions1=[1, 1, 1], predictions2=[1, 0, 1]) +print(results) +{'stat': 1.0, 'p': 0.31731050786291115} +``` + +## Limitations and bias + +The McNemar test is a non-parametric test, so it has relatively few assumptions (basically only that the observations are independent). It should be used to analyze paired nominal data only. + +## Citations + +```bibtex +@article{mcnemar1947note, + title={Note on the sampling error of the difference between correlated proportions or percentages}, + author={McNemar, Quinn}, + journal={Psychometrika}, + volume={12}, + number={2}, + pages={153--157}, + year={1947}, + publisher={Springer-Verlag} +} +``` diff --git a/ST/evaluate/comparisons/mcnemar/app.py b/ST/evaluate/comparisons/mcnemar/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6d5b0970c9d00743c31656a2d5c993402d3e9263 --- /dev/null +++ b/ST/evaluate/comparisons/mcnemar/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mcnemar", module_type="comparison") +launch_gradio_widget(module) diff --git a/ST/evaluate/comparisons/mcnemar/mcnemar.py b/ST/evaluate/comparisons/mcnemar/mcnemar.py new file mode 100644 index 0000000000000000000000000000000000000000..86b85b5e33d74260bb78d83a0225dc004729db5d --- /dev/null +++ b/ST/evaluate/comparisons/mcnemar/mcnemar.py @@ -0,0 +1,98 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""McNemar test for model comparison.""" + +import datasets +from scipy.stats import chi2 + +import evaluate + + +_DESCRIPTION = """ +McNemar's test is a diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: +McNemar = (SE - SP)**2 / SE + SP + Where: +SE: Sensitivity (Test 1 positive; Test 2 negative) +SP: Specificity (Test 1 negative; Test 2 positive) +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `int`): Predicted labels for model 1. + predictions2 (`list` of `int`): Predicted labels for model 2. + references (`list` of `int`): Ground truth labels. + +Returns: + stat (`float`): McNemar test score. + p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference. + +Examples: + >>> mcnemar = evaluate.load("mcnemar") + >>> results = mcnemar.compute(references=[1, 0, 1], predictions1=[1, 1, 1], predictions2=[1, 0, 1]) + >>> print(results) + {'stat': 1.0, 'p': 0.31731050786291115} +""" + + +_CITATION = """ +@article{mcnemar1947note, + title={Note on the sampling error of the difference between correlated proportions or percentages}, + author={McNemar, Quinn}, + journal={Psychometrika}, + volume={12}, + number={2}, + pages={153--157}, + year={1947}, + publisher={Springer-Verlag} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class McNemar(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("int64"), + "predictions2": datasets.Value("int64"), + "references": datasets.Value("int64"), + } + ), + ) + + def _compute(self, predictions1, predictions2, references): + # construct contingency table + tbl = [[0, 0], [0, 0]] + for gt, p1, p2 in zip(references, predictions1, predictions2): + if p1 == gt and p2 == gt: + tbl[0][0] += 1 + elif p1 == gt: + tbl[0][1] += 1 + elif p2 == gt: + tbl[1][0] += 1 + else: + tbl[1][1] += 1 + + # compute statistic + b, c = tbl[0][1], tbl[1][0] + statistic = abs(b - c) ** 2 / (1.0 * (b + c)) + df = 1 + pvalue = chi2.sf(statistic, df) + return {"stat": statistic, "p": pvalue} diff --git a/ST/evaluate/comparisons/mcnemar/requirements.txt b/ST/evaluate/comparisons/mcnemar/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/ST/evaluate/comparisons/mcnemar/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/ST/evaluate/comparisons/wilcoxon/README.md b/ST/evaluate/comparisons/wilcoxon/README.md new file mode 100644 index 0000000000000000000000000000000000000000..17e5248d4a4f49cecada5a8c6719f3bee85d995a --- /dev/null +++ b/ST/evaluate/comparisons/wilcoxon/README.md @@ -0,0 +1,70 @@ +--- +title: Wilcoxon +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + Wilcoxon's test is a signed-rank test for comparing paired samples. +--- + + +# Comparison Card for Wilcoxon + +## Comparison description + +Wilcoxon's test is a non-parametric signed-rank test that tests whether the distribution of the differences is symmetric about zero. It can be used to compare the predictions of two models. + +## How to use + +The Wilcoxon comparison is used to analyze paired ordinal data. + +## Inputs + +Its arguments are: + +`predictions1`: a list of predictions from the first model. + +`predictions2`: a list of predictions from the second model. + +## Output values + +The Wilcoxon comparison outputs two things: + +`stat`: The Wilcoxon statistic. + +`p`: The p value. + +## Examples + +Example comparison: + +```python +wilcoxon = evaluate.load("wilcoxon") +results = wilcoxon.compute(predictions1=[-7, 123.45, 43, 4.91, 5], predictions2=[1337.12, -9.74, 1, 2, 3.21]) +print(results) +{'stat': 5.0, 'p': 0.625} +``` + +## Limitations and bias + +The Wilcoxon test is a non-parametric test, so it has relatively few assumptions (basically only that the observations are independent). It should be used to analyze paired ordinal data only. + +## Citations + +```bibtex +@incollection{wilcoxon1992individual, + title={Individual comparisons by ranking methods}, + author={Wilcoxon, Frank}, + booktitle={Breakthroughs in statistics}, + pages={196--202}, + year={1992}, + publisher={Springer} +} +``` diff --git a/ST/evaluate/comparisons/wilcoxon/app.py b/ST/evaluate/comparisons/wilcoxon/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1c12bf33be18033b4fc086c4db95a426266f595c --- /dev/null +++ b/ST/evaluate/comparisons/wilcoxon/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wilcoxon", module_type="comparison") +launch_gradio_widget(module) diff --git a/ST/evaluate/comparisons/wilcoxon/requirements.txt b/ST/evaluate/comparisons/wilcoxon/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..928da8437c330671424203a5de68732e7cd21a3b --- /dev/null +++ b/ST/evaluate/comparisons/wilcoxon/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@a45df1eb9996eec64ec3282ebe554061cb366388 +datasets~=2.0 +scipy diff --git a/ST/evaluate/comparisons/wilcoxon/wilcoxon.py b/ST/evaluate/comparisons/wilcoxon/wilcoxon.py new file mode 100644 index 0000000000000000000000000000000000000000..a2dc1671b1f7e25f633cd3030a91cf374993c37a --- /dev/null +++ b/ST/evaluate/comparisons/wilcoxon/wilcoxon.py @@ -0,0 +1,78 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Wilcoxon test for model comparison.""" + +import datasets +from scipy.stats import wilcoxon + +import evaluate + + +_DESCRIPTION = """ +Wilcoxon's test is a non-parametric signed-rank test that tests whether the distribution of the differences is symmetric about zero. It can be used to compare the predictions of two models. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `float`): Predictions for model 1. + predictions2 (`list` of `float`): Predictions for model 2. + +Returns: + stat (`float`): Wilcoxon test score. + p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference. + +Examples: + >>> wilcoxon = evaluate.load("wilcoxon") + >>> results = wilcoxon.compute(predictions1=[-7, 123.45, 43, 4.91, 5], predictions2=[1337.12, -9.74, 1, 2, 3.21]) + >>> print(results) + {'stat': 5.0, 'p': 0.625} +""" + + +_CITATION = """ +@incollection{wilcoxon1992individual, + title={Individual comparisons by ranking methods}, + author={Wilcoxon, Frank}, + booktitle={Breakthroughs in statistics}, + pages={196--202}, + year={1992}, + publisher={Springer} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Wilcoxon(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("float"), + "predictions2": datasets.Value("float"), + } + ), + ) + + def _compute(self, predictions1, predictions2): + # calculate difference + d = [p1 - p2 for (p1, p2) in zip(predictions1, predictions2)] + + # compute statistic + res = wilcoxon(d) + return {"stat": res.statistic, "p": res.pvalue} diff --git a/ST/evaluate/docs/README.md b/ST/evaluate/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..62b07e07690f7c26bb1a9e3e0a8c621f665bc5c0 --- /dev/null +++ b/ST/evaluate/docs/README.md @@ -0,0 +1,285 @@ + + +# Generating the documentation + +To generate the documentation, you first have to build it. Several packages are necessary to build the doc, +you can install them with the following command, at the root of the code repository: + +```bash +pip install -e ".[docs]" +``` + +Then you need to install our special tool that builds the documentation: + +```bash +pip install git+https://github.com/huggingface/doc-builder +``` + +--- +**NOTE** + +You only need to generate the documentation to inspect it locally (if you're planning changes and want to +check how they look like before committing for instance). You don't have to commit the built documentation. + +--- + +## Building the documentation + +Once you have setup the `doc-builder` and additional packages, you can generate the documentation by typing th +following command: + +```bash +doc-builder build transformers docs/source/ --build_dir ~/tmp/test-build +``` + +You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate +the MDX files that will be rendered as the documentation on the main website. You can inspect them in your favorite +Markdown editor. + +--- +**NOTE** + +It's not possible to see locally how the final documentation will look like for now. Once you have opened a PR, you +will see a bot add a comment to a link where the documentation with your changes lives. + +--- + +## Adding a new element to the navigation bar + +Accepted files are Markdown (.md or .mdx). + +Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/transformers/blob/master/docs/source/_toctree.yml) file. + +## Renaming section headers and moving sections + +It helps to keep the old links working when renaming section header and/or moving sections from one document to another. This is because the old links are likely to be used in Issues, Forums and Social media and it'd be make for a much more superior user experience if users reading those months later could still easily navigate to the originally intended information. + +Therefore we simply keep a little map of moved sections at the end of the document where the original section was. The key is to preserve the original anchor. + +So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file: + +``` +Sections that were moved: + +[ Section A ] +``` +and of course if you moved it to another file, then: + +``` +Sections that were moved: + +[ Section A ] +``` + +Use the relative style to link to the new file so that the versioned docs continue to work. + +For an example of a rich moved sections set please see the very end of [the Trainer doc](https://github.com/huggingface/transformers/blob/master/docs/source/main_classes/trainer.mdx). + + +## Writing Documentation - Specification + +The `huggingface/transformers` documentation follows the +[Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, +although we can write them directly in Markdown. + +### Adding a new tutorial + +Adding a new tutorial or section is done in two steps: + +- Add a new file under `./source`. This file can either be ReStructuredText (.rst) or Markdown (.md). +- Link that file in `./source/_toctree.yml` on the correct toc-tree. + +Make sure to put your new file under the proper section. It's unlikely to go in the first section (*Get Started*), so +depending on the intended targets (beginners, more advanced users or researchers) it should go in section two, three or +four. + +### Adding a new model + +When adding a new model: + +- Create a file `xxx.mdx` or under `./source/model_doc` (don't hesitate to copy an existing file as template). +- Link that file in `./source/_toctree.yml`. +- Write a short overview of the model: + - Overview with paper & authors + - Paper abstract + - Tips and tricks and how to use it best +- Add the classes that should be linked in the model. This generally includes the configuration, the tokenizer, and + every model of that class (the base model, alongside models with additional heads), both in PyTorch and TensorFlow. + The order is generally: + - Configuration, + - Tokenizer + - PyTorch base model + - PyTorch head models + - TensorFlow base model + - TensorFlow head models + - Flax base model + - Flax head models + +These classes should be added using our Markdown syntax. Usually as follows: + +``` +## XXXConfig + +[[autodoc]] XXXConfig +``` + +This will include every public method of the configuration that is documented. If for some reason you wish for a method +not to be displayed in the documentation, you can do so by specifying which methods should be in the docs: + +``` +## XXXTokenizer + +[[autodoc]] XXXTokenizer + - build_inputs_with_special_tokens + - get_special_tokens_mask + - create_token_type_ids_from_sequences + - save_vocabulary +``` + +If you just want to add a method that is not documented (for instance magic method like `__call__` are not documented +byt default) you can put the list of methods to add in a list that contains `all`: + +``` +## XXXTokenizer + +[[autodoc]] XXXTokenizer + - all + - __call__ +``` + +### Writing source documentation + +Values that should be put in `code` should either be surrounded by backticks: \`like so\`. Note that argument names +and objects like True, None or any strings should usually be put in `code`. + +When mentioning a class, function or method, it is recommended to use our syntax for internal links so that our tool +adds a link to its documentation with this syntax: \[\`XXXClass\`\] or \[\`function\`\]. This requires the class or +function to be in the main package. + +If you want to create a link to some internal class or function, you need to +provide its path. For instance: \[\`file_utils.ModelOutput\`\]. This will be converted into a link with +`file_utils.ModelOutput` in the description. To get rid of the path and only keep the name of the object you are +linking to in the description, add a ~: \[\`~file_utils.ModelOutput\`\] will generate a link with `ModelOutput` in the description. + +The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\]. + +#### Defining arguments in a method + +Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) prefix, followed by a line return and +an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon and its +description: + +``` + Args: + n_layers (`int`): The number of layers of the model. +``` + +If the description is too long to fit in one line, another indentation is necessary before writing the description +after th argument. + +Here's an example showcasing everything so far: + +``` + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AlbertTokenizer`]. See [`~PreTrainedTokenizer.encode`] and + [`~PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) +``` + +For optional arguments or arguments with defaults we follow the following syntax: imagine we have a function with the +following signature: + +``` +def my_function(x: str = None, a: float = 1): +``` + +then its documentation should look like this: + +``` + Args: + x (`str`, *optional*): + This argument controls ... + a (`float`, *optional*, defaults to 1): + This argument is used to ... +``` + +Note that we always omit the "defaults to \`None\`" when None is the default for any argument. Also note that even +if the first line describing your argument type and its default gets long, you can't break it on several lines. You can +however write as many lines as you want in the indented description (see the example above with `input_ids`). + +#### Writing a multi-line code block + +Multi-line code blocks can be useful for displaying examples. They are done between two lines of three backticks as usual in Markdown: + + +```` +``` +# first line of code +# second line +# etc +``` +```` + +We follow the [doctest](https://docs.python.org/3/library/doctest.html) syntax for the examples to automatically test +the results stay consistent with the library. + +#### Writing a return block + +The return block should be introduced with the `Returns:` prefix, followed by a line return and an indentation. +The first line should be the type of the return, followed by a line return. No need to indent further for the elements +building the return. + +Here's an example for a single value return: + +``` + Returns: + `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token. +``` + +Here's an example for tuple return, comprising several objects: + +``` + Returns: + `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs: + - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` -- + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + - **prediction_scores** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) -- + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). +``` + +#### Adding an image + +Due to the rapidly growing repository, it is important to make sure that no files that would significantly weigh down the repository are added. This includes images, videos and other non-text files. We prefer to leverage a hf.co hosted `dataset` like +the ones hosted on [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) in which to place these files and reference +them by URL. We recommend putting them in the following dataset: [huggingface/documentation-images](https://huggingface.co/datasets/huggingface/documentation-images). +If an external contribution, feel free to add the images to your PR and ask a Hugging Face member to migrate your images +to this dataset. + +## Styling the docstring + +We have an automatic script running with the `make style` comment that will make sure that: +- the docstrings fully take advantage of the line width +- all code examples are formatted using black, like the code of the Transformers library + +This script may have some weird failures if you made a syntax mistake or if you uncover a bug. Therefore, it's +recommended to commit your changes before running `make style`, so you can revert the changes done by that script +easily. diff --git a/ST/evaluate/docs/source/_toctree.yml b/ST/evaluate/docs/source/_toctree.yml new file mode 100644 index 0000000000000000000000000000000000000000..a79f3a43f4372795cc73394897d8efd834933f37 --- /dev/null +++ b/ST/evaluate/docs/source/_toctree.yml @@ -0,0 +1,52 @@ +- sections: + - local: index + title: 🤗 Evaluate + title: Get started +- sections: + - local: installation + title: Installation + - local: a_quick_tour + title: A quick tour + title: Tutorials +- sections: + - local: choosing_a_metric + title: Choosing the right metric + - local: creating_and_sharing + title: Adding new evaluations + - local: base_evaluator + title: Using the evaluator + - local: custom_evaluator + title: Using the evaluator with custom pipelines + - local: evaluation_suite + title: Creating an EvaluationSuite + - sections: + - local: transformers_integrations + title: Transformers + - local: keras_integrations + title: Keras and Tensorflow + - local: sklearn_integrations + title: scikit-learn + title: Using 🤗 Evaluate with other ML frameworks + title: "How-to guides" +- sections: + - local: types_of_evaluations + title: Types of evaluations + - local: considerations + title: Considerations for model evaluation + title: "Conceptual guides" +- sections: + - local: package_reference/main_classes + title: Main classes + - local: package_reference/loading_methods + title: Loading methods + - local: package_reference/saving_methods + title: Saving methods + - local: package_reference/hub_methods + title: Hub methods + - local: package_reference/evaluator_classes + title: Evaluator classes + - local: package_reference/visualization_methods + title: Visualization methods + - local: package_reference/logging_methods + title: Logging methods + title: "Reference" diff --git a/ST/evaluate/docs/source/a_quick_tour.mdx b/ST/evaluate/docs/source/a_quick_tour.mdx new file mode 100644 index 0000000000000000000000000000000000000000..371f7f117cb83f499345398137c5e69261fb2d80 --- /dev/null +++ b/ST/evaluate/docs/source/a_quick_tour.mdx @@ -0,0 +1,380 @@ +# A quick tour + +🤗 Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories. + +## Types of evaluations + +There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect 🤗 Evaluate provides a tool: + +- **Metric**: A metric is used to evaluate a model's performance and usually involves the model's predictions as well as some ground truth labels. You can find all integrated metrics at [evaluate-metric](https://huggingface.co/evaluate-metric). +- **Comparison**: A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at [evaluate-comparison](https://huggingface.co/evaluate-comparison). +- **Measurement**: The dataset is as important as the model trained on it. With measurements one can investigate a dataset's properties. You can find all integrated measurements at [evaluate-measurement](https://huggingface.co/evaluate-measurement). + +Each of these evaluation modules live on Hugging Face Hub as a Space. They come with an interactive widget and a documentation card documenting its use and limitations. For example [accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy): + +
+ +
+ +Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: [`evaluate.load`]! + +## Load + +Any metric, comparison, or measurement is loaded with the `evaluate.load` function: + +```py +>>> import evaluate +>>> accuracy = evaluate.load("accuracy") +``` + +If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitly pass the type: + +```py +>>> word_length = evaluate.load("word_length", module_type="measurement") +``` + +### Community modules + +Besides the modules implemented in 🤗 Evaluate you can also load any community module by specifying the repository ID of the metric implementation: + +```py +>>> element_count = evaluate.load("lvwerra/element_count", module_type="measurement") +``` + +See the [Creating and Sharing Guide](/docs/evaluate/main/en/creating_and_sharing) for information about uploading custom metrics. + +### List available modules + +With [`list_evaluation_modules`] you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes: + +```python +>>> evaluate.list_evaluation_modules( +... module_type="comparison", +... include_community=False, +... with_details=True) + +[{'name': 'mcnemar', 'type': 'comparison', 'community': False, 'likes': 1}, + {'name': 'exact_match', 'type': 'comparison', 'community': False, 'likes': 0}] +``` + +## Module attributes + +All evalution modules come with a range of useful attributes that help to use a module stored in a [`EvaluationModuleInfo`] object. + +|Attribute|Description| +|---|---| +|`description`|A short description of the evaluation module.| +|`citation`|A BibTex string for citation when available.| +|`features`|A `Features` object defining the input format.| +|`inputs_description`|This is equivalent to the modules docstring.| +|`homepage`|The homepage of the module.| +|`license`|The license of the module.| +|`codebase_urls`|Link to the code behind the module.| +|`reference_urls`|Additional reference URLs.| + +Let's have a look at a few examples. First, let's look at the `description` attribute of the accuracy metric: + +```py +>>> accuracy = evaluate.load("accuracy") +>>> accuracy.description +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative +``` + +You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the `citation` attribute: + +```py +>>> accuracy.citation +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is: + +```py +>>> accuracy.features +{ + 'predictions': Value(dtype='int32', id=None), + 'references': Value(dtype='int32', id=None) +} +``` + + + +Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in `features`. Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation. + + + +## Compute + +Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it: + +1. All-in-one +2. Incremental + + +In the incremental approach the necessary inputs are added to the module with [`EvaluationModule.add`] or [`EvaluationModule.add_batch`] and the score is calculated at the end with [`EvaluationModule.compute`]. Alternatively, one can pass all the inputs at once to `compute()`. Let's have a look at the two approaches. + +### How to compute + +The simplest way to calculate the score of an evaluation module is by calling `compute()` directly with the necessary inputs. Simply pass the inputs as seen in `features` to the `compute()` method. + +```py +>>> accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1]) +{'accuracy': 0.5} +``` +Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case `add()` or `add_batch()` are useful. + +### Calculate a single metric or a batch of metrics + +In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to `compute()`. With `add()` and `add_batch()` you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use `add()`: + +```py +>>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): +>>> accuracy.add(references=ref, predictions=pred) +>>> accuracy.compute() +{'accuracy': 0.5} +``` + +Once you have gathered all predictions you can call `compute()` to compute the score based on all stored values. When getting predictions and references in batches you can use `add_batch()` which adds a list elements for later processing. The rest works as with `add()`: + +```py +>>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): +>>> accuracy.add_batch(references=refs, predictions=preds) +>>> accuracy.compute() +{'accuracy': 0.5} +``` + +This is especially useful when you need to get the predictions from your model in batches: + +```py +>>> for model_inputs, gold_standards in evaluation_dataset: +>>> predictions = model(model_inputs) +>>> metric.add_batch(references=gold_standards, predictions=predictions) +>>> metric.compute() +``` + +### Distributed evaluation + +Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive (`f(AuB) = f(A) + f(B)`), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive (`f(AuB) ≠ f(A) + f(B)`), it's not that simple. For example, you can't take the sum of the [F1](https://huggingface.co/spaces/evaluate-metric/f1) scores of each data subset as your **final metric**. + +A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient. + +🤗 Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to `compute()` the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, `compute()` will perform the final metric evaluation. + +This solution allows 🤗 Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory. + +## Combining several evaluations + +Often one wants to not only evaluate a single metric but a range of different metrics capturing different aspects of a model. E.g. for classification it is usually a good idea to compute F1-score, recall, and precision in addition to accuracy to get a better picture of model performance. Naturally, you can load a bunch of metrics and call them sequentially. However, a more convenient way is to use the [`~evaluate.combine`] function to bundle them together: + +```python +>>> clf_metrics = evaluate.combine(["accuracy", "f1", "precision", "recall"]) +``` + +The `combine` function accepts both the list of names of the metrics as well as an instantiated modules. The `compute` call then computes each metric: + +```python +>>> clf_metrics.compute(predictions=[0, 1, 0], references=[0, 1, 1]) + +{ + 'accuracy': 0.667, + 'f1': 0.667, + 'precision': 1.0, + 'recall': 0.5 +} +``` + +## Save and push to the Hub + +Saving and sharing evaluation results is an important step. We provide the [`evaluate.save`] function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file. + +```py +>>> result = accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1]) + +>>> hyperparams = {"model": "bert-base-uncased"} +>>> evaluate.save("./results/", experiment="run 42", **result, **hyperparams) +PosixPath('results/result-2022_05_30-22_09_11.json') +``` + +The content of the JSON file look like the following: + +```json +{ + "experiment": "run 42", + "accuracy": 0.5, + "model": "bert-base-uncased", + "_timestamp": "2022-05-30T22:09:11.959469", + "_git_commit_hash": "123456789abcdefghijkl", + "_evaluate_version": "0.1.0", + "_python_version": "3.9.12 (main, Mar 26 2022, 15:51:15) \n[Clang 13.1.6 (clang-1316.0.21.2)]", + "_interpreter_path": "/Users/leandro/git/evaluate/env/bin/python" +} +``` + +In addition to the specified fields, it also contains useful system information for reproducing the results. + +Besides storing the results locally, you should report them on the model's repository on the Hub. With the [`evaluate.push_to_hub`] function, you can easily report evaluation results to the model's repository: + +```py +evaluate.push_to_hub( + model_id="huggingface/gpt2-wikitext2", # model repository on hub + metric_value=0.5, # metric value + metric_type="bleu", # metric name, e.g. accuracy.name + metric_name="BLEU", # pretty name which is displayed + dataset_type="wikitext", # dataset name on the hub + dataset_name="WikiText", # pretty name + dataset_split="test", # dataset split used + task_type="text-generation", # task id, see https://github.com/huggingface/evaluate/blob/main/src/evaluate/config.py#L154-L192 + task_name="Text Generation" # pretty name for task +) +``` + +## Evaluator + +The [`evaluate.evaluator`] provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in `EvaluationModule`s that require the model's predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the [`~transformers.pipeline`] abstraction from `transformers`. However, you can use your own framework as long as it follows the `pipeline` interface. + +To make an evaluation with the `evaluator` let's load a `transformers` pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric. + +```python +from transformers import pipeline +from datasets import load_dataset +from evaluate import evaluator +import evaluate + +pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb", device=0) +data = load_dataset("imdb", split="test").shuffle().select(range(1000)) +metric = evaluate.load("accuracy") +``` + +Then you can create an evaluator for text classification and pass the three objects to the `compute()` method. With the label mapping `evaluate` provides a method to align the pipeline outputs with the label column in the dataset: + +```python +>>> task_evaluator = evaluator("text-classification") + +>>> results = task_evaluator.compute(model_or_pipeline=pipe, data=data, metric=metric, +... label_mapping={"NEGATIVE": 0, "POSITIVE": 1},) + +>>> print(results) +{'accuracy': 0.934} +``` + +Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With _bootstrapping_ `evaluate` computes confidence intervals and the standard error which helps estimate how stable a score is: + +```python +>>> results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric, +... label_mapping={"NEGATIVE": 0, "POSITIVE": 1}, +... strategy="bootstrap", n_resamples=200) + +>>> print(results) +{'accuracy': + { + 'confidence_interval': (0.906, 0.9406749892841922), + 'standard_error': 0.00865213251082787, + 'score': 0.923 + } +} +``` + +The evaluator expects a `"text"` and `"label"` column for the data input. If your dataset differs you can provide the columns with the keywords `input_column="text"` and `label_column="label"`. Currently only `"text-classification"` is supported with more tasks being added in the future. + + +## Visualization + +When comparing several models, sometimes it's hard to spot the differences in their performance simply by looking at their scores. Also often there is not a single best model but there are trade-offs between e.g. latency and accuracy as larger models might have better performance but are also slower. We are gradually adding different visualization approaches, such as plots, to make choosing the best model for a use-case easier. + +For instance, if you have a list of results from multiple models (as dictionaries), you can feed them into the `radar_plot()` function: + +```python +import evaluate +from evaluate.visualization import radar_plot + +>>> data = [ + {"accuracy": 0.99, "precision": 0.8, "f1": 0.95, "latency_in_seconds": 33.6}, + {"accuracy": 0.98, "precision": 0.87, "f1": 0.91, "latency_in_seconds": 11.2}, + {"accuracy": 0.98, "precision": 0.78, "f1": 0.88, "latency_in_seconds": 87.6}, + {"accuracy": 0.88, "precision": 0.78, "f1": 0.81, "latency_in_seconds": 101.6} + ] +>>> model_names = ["Model 1", "Model 2", "Model 3", "Model 4"] +>>> plot = radar_plot(data=data, model_names=model_names) +>>> plot.show() +``` + +Which lets you visually compare the 4 models and choose the optimal one for you, based on one or several metrics: +
+ +
+ +## Running evaluation on a suite of tasks + +It can be useful to evaluate models on a variety of different tasks to understand their downstream performance. The [EvaluationSuite](evaluation_suite) enables evaluation of models on a collection of tasks. Tasks can be constructed as ([evaluator](base_evaluator), dataset, metric) tuples and passed to an [EvaluationSuite](evaluation_suite) stored on the Hugging Face Hub as a Space, or locally as a Python script. See the [evaluator documentation](base_evaluator) for a list of currently supported tasks. + +`EvaluationSuite` scripts can be defined as follows, and supports Python code for data preprocessing. + +```python +import evaluate +from evaluate.evaluation_suite import SubTask + +class Suite(evaluate.EvaluationSuite): + + def __init__(self, name): + super().__init__(name) + + self.suite = [ + SubTask( + task_type="text-classification", + data="imdb", + split="test[:1]", + args_for_task={ + "metric": "accuracy", + "input_column": "text", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ), + SubTask( + task_type="text-classification", + data="sst2", + split="test[:1]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ) + ] +``` + +Evaluation can be run by loading the `EvaluationSuite` and calling the `run()` method with a model or pipeline. + +``` +>>> from evaluate import EvaluationSuite +>>> suite = EvaluationSuite.load('mathemakitten/sentiment-evaluation-suite') +>>> results = suite.run("huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli") +``` + +| accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | task_name | +|------------:|---------------------:|--------------------------:|:----------------|:-----------| +| 0.3 | 4.62804 | 2.16074 | 0.462804 | imdb | +| 0 | 0.686388 | 14.569 | 0.0686388 | sst2 | diff --git a/ST/evaluate/docs/source/base_evaluator.mdx b/ST/evaluate/docs/source/base_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..043c825189d3ae8616a141c90d9ebd0fb1434ba7 --- /dev/null +++ b/ST/evaluate/docs/source/base_evaluator.mdx @@ -0,0 +1,294 @@ +# Using the `evaluator` + +The `Evaluator` classes allow to evaluate a triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, `Evaluator`s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section [Using the `evaluator` with custom pipelines](custom_evaluator). + +Currently supported tasks are: +- `"text-classification"`: will use the [`TextClassificationEvaluator`]. +- `"token-classification"`: will use the [`TokenClassificationEvaluator`]. +- `"question-answering"`: will use the [`QuestionAnsweringEvaluator`]. +- `"image-classification"`: will use the [`ImageClassificationEvaluator`]. +- `"text-generation"`: will use the [`TextGenerationEvaluator`]. +- `"text2text-generation"`: will use the [`Text2TextGenerationEvaluator`]. +- `"summarization"`: will use the [`SummarizationEvaluator`]. +- `"translation"`: will use the [`TranslationEvaluator`]. +- `"automatic-speech-recognition"`: will use the [`AutomaticSpeechRecognitionEvaluator`]. +- `"audio-classification"`: will use the [`AudioClassificationEvaluator`]. + +To run an `Evaluator` with several tasks in a single call, use the [EvaluationSuite](evaluation_suite), which runs evaluations on a collection of `SubTask`s. + +Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let's have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time. + +## Text classification + +The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs: + +- `input_column="text"`: with this argument the column with the data for the pipeline can be specified. +- `label_column="label"`: with this argument the column with the labels for the evaluation can be specified. +- `label_mapping=None`: the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in `label_column` can be integers (`0`/`1`) whereas the pipeline can produce label names such as `"positive"`/`"negative"`. With that dictionary the pipeline outputs are mapped to the labels. + +By default the `"accuracy"` metric is computed. + +### Evaluate models on the Hub + +There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a `transformers` model and pass it to the evaluator or you can pass an initialized `transformers.Pipeline`. Alternatively you can pass any callable function that behaves like a `pipeline` call for the task in any framework. + +So any of the following works: + +```py +from datasets import load_dataset +from evaluate import evaluator +from transformers import AutoModelForSequenceClassification, pipeline + +data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(1000)) +task_evaluator = evaluator("text-classification") + +# 1. Pass a model name or path +eval_results = task_evaluator.compute( + model_or_pipeline="lvwerra/distilbert-imdb", + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +# 2. Pass an instantiated model +model = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb") + +eval_results = task_evaluator.compute( + model_or_pipeline=model, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +# 3. Pass an instantiated pipeline +pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb") + +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) +print(eval_results) +``` + + +Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass `device` to `compute` where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device. + + + + +The results will look as follows: +```python +{ + 'accuracy': 0.918, + 'latency_in_seconds': 0.013, + 'samples_per_second': 78.887, + 'total_time_in_seconds': 12.676 +} +``` + +Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline. + + + +The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size. + + + +### Evaluate multiple metrics + +With the [`combine`] function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator: + +```python +import evaluate + +eval_results = task_evaluator.compute( + model_or_pipeline="lvwerra/distilbert-imdb", + data=data, + metric=evaluate.combine(["accuracy", "recall", "precision", "f1"]), + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) +print(eval_results) + +``` +The results will look as follows: +```python +{ + 'accuracy': 0.918, + 'f1': 0.916, + 'precision': 0.9147, + 'recall': 0.9187, + 'latency_in_seconds': 0.013, + 'samples_per_second': 78.887, + 'total_time_in_seconds': 12.676 +} +``` + +Next let's have a look at token classification. + +## Token Classification + +With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments: + +- `input_column="text"`: with this argument the column with the data for the pipeline can be specified. +- `label_column="label"`: with this argument the column with the labels for the evaluation can be specified. +- `label_mapping=None`: the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in `label_column` can be integers (`0`/`1`) whereas the pipeline can produce label names such as `"positive"`/`"negative"`. With that dictionary the pipeline outputs are mapped to the labels. +- `join_by=" "`: While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace. + +Let's have a look how we can use the evaluator to benchmark several models. + +### Benchmarking several models + +Here is an example where several models can be compared thanks to the `evaluator` in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation: + +```python +import pandas as pd +from datasets import load_dataset +from evaluate import evaluator +from transformers import pipeline + +models = [ + "xlm-roberta-large-finetuned-conll03-english", + "dbmdz/bert-large-cased-finetuned-conll03-english", + "elastic/distilbert-base-uncased-finetuned-conll03-english", + "dbmdz/electra-large-discriminator-finetuned-conll03-english", + "gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner", + "philschmid/distilroberta-base-ner-conll2003", + "Jorgeutd/albert-base-v2-finetuned-ner", +] + +data = load_dataset("conll2003", split="validation").shuffle().select(range(1000)) +task_evaluator = evaluator("token-classification") + +results = [] +for model in models: + results.append( + task_evaluator.compute( + model_or_pipeline=model, data=data, metric="seqeval" + ) + ) + +df = pd.DataFrame(results, index=models) +df[["overall_f1", "overall_accuracy", "total_time_in_seconds", "samples_per_second", "latency_in_seconds"]] +``` + +The result is a table that looks like this: + +| model | overall_f1 | overall_accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | +|:-------------------------------------------------------------------|-------------:|-------------------:|------------------------:|---------------------:|---------------------:| +| Jorgeutd/albert-base-v2-finetuned-ner | 0.941 | 0.989 | 4.515 | 221.468 | 0.005 | +| dbmdz/bert-large-cased-finetuned-conll03-english | 0.962 | 0.881 | 11.648 | 85.850 | 0.012 | +| dbmdz/electra-large-discriminator-finetuned-conll03-english | 0.965 | 0.881 | 11.456 | 87.292 | 0.011 | +| elastic/distilbert-base-uncased-finetuned-conll03-english | 0.940 | 0.989 | 2.318 | 431.378 | 0.002 | +| gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner | 0.947 | 0.991 | 2.376 | 420.873 | 0.002 | +| philschmid/distilroberta-base-ner-conll2003 | 0.961 | 0.994 | 2.436 | 410.579 | 0.002 | +| xlm-roberta-large-finetuned-conll03-english | 0.969 | 0.882 | 11.996 | 83.359 | 0.012 | + + +### Visualizing results + +You can feed in the `results` list above into the `plot_radar()` function to visualize different aspects of their performance and choose the model that is the best fit, depending on the metric(s) that are relevant to your use case: + +```python +import evaluate +from evaluate.visualization import radar_plot + +>>> plot = radar_plot(data=results, model_names=models, invert_range=["latency_in_seconds"]) +>>> plot.show() +``` + +
+ +
+ + +Don't forget to specify `invert_range` for metrics for which smaller is better (such as the case for latency in seconds). + +If you want to save the plot locally, you can use the `plot.savefig()` function with the option `bbox_inches='tight'`, to make sure no part of the image gets cut off. + + +## Question Answering + +With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that's required for these models. It has the following specific arguments: + + +- `question_column="question"`: the name of the column containing the question in the dataset +- `context_column="context"`: the name of the column containing the context +- `id_column="id"`: the name of the column cointaing the identification field of the question and answer pair +- `label_column="answers"`: the name of the column containing the answers +- `squad_v2_format=None`: whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred. + +Let's have a look how we can evaluate QA models and compute confidence intervals at the same time. + +### Confidence intervals + +Every evaluator comes with the options to compute confidence intervals using [bootstrapping](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html). Simply pass `strategy="bootstrap"` and set the number of resanmples with `n_resamples`. + +```python +from datasets import load_dataset +from evaluate import evaluator + +task_evaluator = evaluator("question-answering") + +data = load_dataset("squad", split="validation[:1000]") +eval_results = task_evaluator.compute( + model_or_pipeline="distilbert-base-uncased-distilled-squad", + data=data, + metric="squad", + strategy="bootstrap", + n_resamples=30 +) +``` + +Results include confidence intervals as well as error estimates as follows: + +```python +{ + 'exact_match': + { + 'confidence_interval': (79.67, 84.54), + 'score': 82.30, + 'standard_error': 1.28 + }, + 'f1': + { + 'confidence_interval': (85.30, 88.88), + 'score': 87.23, + 'standard_error': 0.97 + }, + 'latency_in_seconds': 0.0085, + 'samples_per_second': 117.31, + 'total_time_in_seconds': 8.52 + } +``` + +## Image classification + +With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier: + +- `input_column="image"`: the name of the column containing the images as PIL ImageFile +- `label_column="label"`: the name of the column containing the labels +- `label_mapping=None`: We want to map class labels defined by the model in the pipeline to values consistent with those defined in the `label_column` + +Let's have a look at how can evaluate image classification models on large datasets. + +### Handling large datasets + +The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB. + +```python +data = load_dataset("imagenet-1k", split="validation", use_auth_token=True) + +pipe = pipeline( + task="image-classification", + model="facebook/deit-small-distilled-patch16-224" +) + +task_evaluator = evaluator("image-classification") +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + metric="accuracy", + label_mapping=pipe.model.config.label2id +) +``` + +Since we are using `datasets` to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big. diff --git a/ST/evaluate/docs/source/choosing_a_metric.mdx b/ST/evaluate/docs/source/choosing_a_metric.mdx new file mode 100644 index 0000000000000000000000000000000000000000..344f830586a8cb78fefd327d1de1472c7d3b02c3 --- /dev/null +++ b/ST/evaluate/docs/source/choosing_a_metric.mdx @@ -0,0 +1,64 @@ +# Choosing a metric for your task + +**So you've trained your model and want to see how well it’s doing on a dataset of your choice. Where do you start?** + +There is no “one size fits all” approach to choosing an evaluation metric, but some good guidelines to keep in mind are: + +## Categories of metrics + +There are 3 high-level categories of metrics: + +1. *Generic metrics*, which can be applied to a variety of situations and datasets, such as precision and accuracy. +2. *Task-specific metrics*, which are limited to a given task, such as Machine Translation (often evaluated using metrics [BLEU](https://huggingface.co/metrics/bleu) or [ROUGE](https://huggingface.co/metrics/rouge)) or Named Entity Recognition (often evaluated with [seqeval](https://huggingface.co/metrics/seqeval)). +3. *Dataset-specific metrics*, which aim to measure model performance on specific benchmarks: for instance, the [GLUE benchmark](https://huggingface.co/datasets/glue) has a dedicated [evaluation metric](https://huggingface.co/metrics/glue). + +Let's look at each of these three cases: + +### Generic metrics + +Many of the metrics used in the Machine Learning community are quite generic and can be applied in a variety of tasks and datasets. + +This is the case for metrics like [accuracy](https://huggingface.co/metrics/accuracy) and [precision](https://huggingface.co/metrics/precision), which can be used for evaluating labeled (supervised) datasets, as well as [perplexity](https://huggingface.co/metrics/perplexity), which can be used for evaluating different kinds of (unsupervised) generative tasks. + +To see the input structure of a given metric, you can look at its metric card. For example, in the case of [precision](https://huggingface.co/metrics/precision), the format is: +``` +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'precision': 1.0} +``` + +### Task-specific metrics + +Popular ML tasks like Machine Translation and Named Entity Recognition have specific metrics that can be used to compare models. For example, a series of different metrics have been proposed for text generation, ranging from [BLEU](https://huggingface.co/metrics/bleu) and its derivatives such as [GoogleBLEU](https://huggingface.co/metrics/google_bleu) and [GLEU](https://huggingface.co/metrics/gleu), but also [ROUGE](https://huggingface.co/metrics/rouge), [MAUVE](https://huggingface.co/metrics/mauve), etc. + +You can find the right metric for your task by: + +- **Looking at the [Task pages](https://huggingface.co/tasks)** to see what metrics can be used for evaluating models for a given task. +- **Checking out leaderboards** on sites like [Papers With Code](https://paperswithcode.com/) (you can search by task and by dataset). +- **Reading the metric cards** for the relevant metrics and see which ones are a good fit for your use case. For example, see the [BLEU metric card](https://github.com/huggingface/evaluate/tree/main/metrics/bleu) or [SQuaD metric card](https://github.com/huggingface/evaluate/tree/main/metrics/squad). +- **Looking at papers and blog posts** published on the topic and see what metrics they report. This can change over time, so try to pick papers from the last couple of years! + +### Dataset-specific metrics + +Some datasets have specific metrics associated with them -- this is especially in the case of popular benchmarks like [GLUE](https://huggingface.co/metrics/glue) and [SQuAD](https://huggingface.co/metrics/squad). + + +💡 +GLUE is actually a collection of different subsets on different tasks, so first you need to choose the one that corresponds to the NLI task, such as mnli, which is described as “crowdsourced collection of sentence pairs with textual entailment annotations” + + + +If you are evaluating your model on a benchmark dataset like the ones mentioned above, you can use its dedicated evaluation metric. Make sure you respect the format that they require. For example, to evaluate your model on the [SQuAD](https://huggingface.co/datasets/squad) dataset, you need to feed the `question` and `context` into your model and return the `prediction_text`, which should be compared with the `references` (based on matching the `id` of the question) : + +``` +>>> from evaluate import load +>>> squad_metric = load("squad") +>>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] +>>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +>>> results = squad_metric.compute(predictions=predictions, references=references) +>>> results +{'exact_match': 100.0, 'f1': 100.0} +``` + +You can find examples of dataset structures by consulting the "Dataset Preview" function or the dataset card for a given dataset, and you can see how to use its dedicated evaluation function based on the metric card. diff --git a/ST/evaluate/docs/source/considerations.mdx b/ST/evaluate/docs/source/considerations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..dc8ca5ccd141b983432e7b094e7a377fe3dcb123 --- /dev/null +++ b/ST/evaluate/docs/source/considerations.mdx @@ -0,0 +1,88 @@ +# Considerations for model evaluation + +Developing an ML model is rarely a one-shot deal: it often involves multiple stages of defining the model architecture and tuning hyper-parameters before converging on a final set. Responsible model evaluation is a key part of this process, and 🤗 Evaluate is here to help! + +Here are some things to keep in mind when evaluating your model using the 🤗 Evaluate library: + +## Properly splitting your data + +Good evaluation generally requires three splits of your dataset: + +- **train**: this is used for training your model. +- **validation**: this is used for validating the model hyperparameters. +- **test**: this is used for evaluating your model. + +Many of the datasets on the 🤗 Hub are separated into 2 splits: `train` and `validation`; others are split into 3 splits (`train`, `validation` and `test`) -- make sure to use the right split for the right purpose! + +Some datasets on the 🤗 Hub are already separated into these three splits. However, there are also many that only have a train/validation or only train split. + +If the dataset you're using doesn't have a predefined train-test split, it is up to you to define which part of the dataset you want to use for training your model and which you want to use for hyperparameter tuning or final evaluation. + + +Training and evaluating on the same split can misrepresent your results! If you overfit on your training data the evaluation results on that split will look great but the model will perform poorly on new data. + + +Depending on the size of the dataset, you can keep anywhere from 10-30% for evaluation and the rest for training, while aiming to set up the test set to reflect the production data as close as possible. Check out [this thread](https://discuss.huggingface.co/t/how-to-split-main-dataset-into-train-dev-test-as-datasetdict/1090) for a more in-depth discussion of dataset splitting! + +## The impact of class imbalance + +While many academic datasets, such as the [IMDb dataset](https://huggingface.co/datasets/imdb) of movie reviews, are perfectly balanced, most real-world datasets are not. In machine learning a *balanced dataset* corresponds to a datasets where all labels are represented equally. In the case of the IMDb dataset this means that there are as many positive as negative reviews in the dataset. In an imbalanced dataset this is not the case: in fraud detection for example there are usually many more non-fraud cases than fraud cases in the dataset. + +Having an imbalanced dataset can skew the results of your metrics. Imagine a dataset with 99 "non-fraud" cases and 1 "fraud" case. A simple model that always predicts "non-fraud" cases would give yield a 99% accuracy which might sound good at first until you realize that you will never catch a fraud case. + +Often, using more than one metric can help get a better idea of your model’s performance from different points of view. For instance, metrics like **[recall](https://huggingface.co/metrics/recall)** and **[precision](https://huggingface.co/metrics/precision)** can be used together, and the **[f1 score](https://huggingface.co/metrics/f1)** is actually the harmonic mean of the two. + +In cases where a dataset is balanced, using [accuracy](https://huggingface.co/metrics/accuracy) can reflect the overall model performance: + +![Balanced Labels](https://huggingface.co/datasets/evaluate/media/resolve/main/balanced-classes.png) + +In cases where there is an imbalance, using [F1 score](https://huggingface.co/metrics/f1) can be a better representation of performance, given that it encompasses both precision and recall. + +![Imbalanced Labels](https://huggingface.co/datasets/evaluate/media/resolve/main/imbalanced-classes.png) + +Using accuracy in an imbalanced setting is less ideal, since it is not sensitive to minority classes and will not faithfully reflect model performance on them. + +## Offline vs. online model evaluation + +There are multiple ways to evaluate models, and an important distinction is offline versus online evaluation: + +**Offline evaluation** is done before deploying a model or using insights generated from a model, using static datasets and metrics. + +**Online evaluation** means evaluating how a model is performing after deployment and during its use in production. + +These two types of evaluation can use different metrics and measure different aspects of model performance. For example, offline evaluation can compare a model to other models based on their performance on common benchmarks, whereas online evaluation will evaluate aspects such as latency and accuracy of the model based on production data (for example, the number of user queries that it was able to address). + +## Trade-offs in model evaluation + +When evaluating models in practice, there are often trade-offs that have to be made between different aspects of model performance: for instance, choosing a model that is slightly less accurate but that has a faster inference time, compared to a high-accuracy that has a higher memory footprint and requires access to more GPUs. + +Here are other aspects of model performance to consider during evaluation: + +### Interpretability + +When evaluating models, **interpretability** (i.e. the ability to *interpret* results) can be very important, especially when deploying models in production. + +For instance, metrics such as [exact match](https://huggingface.co/spaces/evaluate-metric/exact_match) have a set range (between 0 and 1, or 0% and 100%) and are easily understandable to users: for a pair of strings, the exact match score is 1 if the two strings are the exact same, and 0 otherwise. + +Other metrics, such as [BLEU](https://huggingface.co/spaces/evaluate-metric/exact_match) are harder to interpret: while they also range between 0 and 1, they can vary greatly depending on which parameters are used to generate the scores, especially when different tokenization and normalization techniques are used (see the [metric card](https://huggingface.co/spaces/evaluate-metric/bleu/blob/main/README.md) for more information about BLEU limitations). This means that it is difficult to interpret a BLEU score without having more information about the procedure used for obtaining it. + +Interpretability can be more or less important depending on the evaluation use case, but it is a useful aspect of model evaluation to keep in mind, since communicating and comparing model evaluations is an important part of responsible machine learning. + + +### Inference speed and memory footprint + +While recent years have seen increasingly large ML models achieve high performance on a large variety of tasks and benchmarks, deploying these multi-billion parameter models in practice can be a challenge in itself, and many organizations lack the resources for this. This is why considering the **inference speed** and **memory footprint** of models is important, especially when doing online model evaluation. + +Inference speed refers to the time that it takes for a model to make a prediction -- this will vary depending on the hardware used and the way in which models are queried, e.g. in real time via an API or in batch jobs that run once a day. + +Memory footprint refers to the size of the model weights and how much hardware memory they occupy. If a model is too large to fit on a single GPU or CPU, then it has to be split over multiple ones, which can be more or less difficult depending on the model architecture and the deployment method. + +When doing online model evaluation, there is often a trade-off to be done between inference speed and accuracy or precision, whereas this is less the case for offline evaluation. + +## Limitations and bias + +All models and all metrics have their limitations and biases, which depend on the way in which they were trained, the data that was used, and their intended uses. It is important to measure and communicate these limitations clearly to prevent misuse and unintended impacts, for instance via [model cards](https://huggingface.co/course/chapter4/4?fw=pt) which document the training and evaluation process. + +Measuring biases can be done by evaluating models on datasets such as [Wino Bias](https://huggingface.co/datasets/wino_bias) or [MD Gender Bias](https://huggingface.co/datasets/md_gender_bias), and by doing [Interactive Error Analyis](https://huggingface.co/spaces/nazneen/error-analysis) to try to identify which subsets of the evaluation dataset a model performs poorly on. + +We are currently working on additional measurements that can be used to quantify different dimensions of bias in both models and datasets -- stay tuned for more documentation on this topic! diff --git a/ST/evaluate/docs/source/creating_and_sharing.mdx b/ST/evaluate/docs/source/creating_and_sharing.mdx new file mode 100644 index 0000000000000000000000000000000000000000..c1b96d3fcc64c1fd5be5afa6a2dc26ed602e6d83 --- /dev/null +++ b/ST/evaluate/docs/source/creating_and_sharing.mdx @@ -0,0 +1,113 @@ +# Creating and sharing a new evaluation + +## Setup + +Before you can create a new metric make sure you have all the necessary dependencies installed: + +```bash +pip install evaluate[template] +``` + +Also make sure your Hugging Face token is registered so you can connect to the Hugging Face Hub: + +```bash +huggingface-cli login +``` + +## Create + +All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). In principle, you could setup a new Space and add a new module following the same structure. However, we added a CLI that makes creating a new evaluation module much easier: + +```bash +evaluate-cli create "My Metric" --module_type "metric" +``` + +This will create a new Space on the 🤗 Hub, clone it locally, and populate it with a template. Instructions on how to fill the template will be displayed in the terminal, but are also explained here in more detail. + +For more information about Spaces, see the [Spaces documentation](https://huggingface.co/docs/hub/spaces). + +## Module script + +The evaluation module script (the file with suffix `*.py`) is the core of the new module and includes all the code for computing the evaluation. + +### Attributes + +Start by adding some information about your evalution module in [`EvaluationModule._info`]. The most important attributes you should specify are: + +1. [`EvaluationModuleInfo.description`] provides a brief description about your evalution module. + +2. [`EvaluationModuleInfo.citation`] contains a BibTex citation for the evalution module. + +3. [`EvaluationModuleInfo.inputs_description`] describes the expected inputs and outputs. It may also provide an example usage of the evalution module. + +4. [`EvaluationModuleInfo.features`] defines the name and type of the predictions and references. This has to be either a single `datasets.Features` object or a list of `datasets.Features` objects if multiple input types are allowed. + +Then, we can move on to prepare everything before the actual computation. + +### Download + +Some evaluation modules require some external data such as NLTK that requires resources or the BLEURT metric that requires checkpoints. You can implement these downloads in [`EvaluationModule._download_and_prepare`], which downloads and caches the resources via the `dlmanager`. A simplified example on how BLEURT downloads and loads a checkpoint: + +```py +def _download_and_prepare(self, dl_manager): + model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name]) + self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name)) +``` + +Or if you need to download the NLTK `"punkt"` resources: + +```py +def _download_and_prepare(self, dl_manager): + import nltk + nltk.download("punkt") +``` + +Next, we need to define how the computation of the evaluation module works. + +### Compute + +The computation is performed in the [`EvaluationModule._compute`] method. It takes the same arguments as `EvaluationModuleInfo.features` and should then return the result as a dictionary. Here an example of an exact match metric: + + +```py +def _compute(self, references, predictions): + em = sum([r==p for r, p in zip(references, predictions)])/len(references) + return {"exact_match": em} +``` + +This method is used when you call `.compute()` later on. + +## Readme + +When you use the `evalute-cli` to setup the evaluation module the Readme structure and instructions are automatically created. It should include a general description of the metric, information about its input/output format, examples as well as information about its limiations or biases and references. + +## Requirements + +If your evaluation modules has additional dependencies (e.g. `sklearn` or `nltk`) the `requirements.txt` files is the place to put them. The file follows the `pip` format and you can list all dependencies there. + +## App + +The `app.py` is where the Spaces widget lives. In general it looks like the following and does not require any changes: + +```py +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("lvwerra/element_count") +launch_gradio_widget(module) +``` + +If you want a custom widget you could add your gradio app here. + +## Push to Hub + +Finally, when you are done with all the above changes it is time to push your evaluation module to the hub. To do so navigate to the folder of your module and git add/commit/push the changes to the hub: + +``` +cd PATH_TO_MODULE +git add . +git commit -m "Add my new, shiny module." +git push +``` +Tada 🎉! Your evaluation module is now on the 🤗 Hub and ready to be used by everybody! diff --git a/ST/evaluate/docs/source/custom_evaluator.mdx b/ST/evaluate/docs/source/custom_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..288e17e8104b7578b23cb20bc71ade3f07f53442 --- /dev/null +++ b/ST/evaluate/docs/source/custom_evaluator.mdx @@ -0,0 +1,114 @@ +# Using the `evaluator` with custom pipelines + +The evaluator is designed to work with `transformer` pipelines out-of-the-box. However, in many cases you might have a model or pipeline that's not part of the `transformer` ecosystem. You can still use `evaluator` to easily compute metrics for them. In this guide we show how to do this for a Scikit-Learn [pipeline](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline) and a Spacy [pipeline](https://spacy.io). Let's start with the Scikit-Learn case. + +## Scikit-Learn + +First we need to train a model. We'll train a simple text classifier on the [IMDb dataset](https://huggingface.co/datasets/imdb), so let's start by downloading the dataset: + +```py +from datasets import load_dataset + +ds = load_dataset("imdb") +``` + +Then we can build a simple TF-IDF preprocessor and Naive Bayes classifier wrapped in a `Pipeline`: + +```py +from sklearn.pipeline import Pipeline +from sklearn.naive_bayes import MultinomialNB +from sklearn.feature_extraction.text import TfidfTransformer +from sklearn.feature_extraction.text import CountVectorizer + +text_clf = Pipeline([ + ('vect', CountVectorizer()), + ('tfidf', TfidfTransformer()), + ('clf', MultinomialNB()), +]) + +text_clf.fit(ds["train"]["text"], ds["train"]["label"]) +``` + +Following the convention in the `TextClassificationPipeline` of `transformers` our pipeline should be callable and return a list of dictionaries. In addition we use the `task` attribute to check if the pipeline is compatible with the `evaluator`. We can write a small wrapper class for that purpose: + +```py +class ScikitEvalPipeline: + def __init__(self, pipeline): + self.pipeline = pipeline + self.task = "text-classification" + + def __call__(self, input_texts, **kwargs): + return [{"label": p} for p in self.pipeline.predict(input_texts)] + +pipe = ScikitEvalPipeline(text_clf) +``` + +We can now pass this `pipeline` to the `evaluator`: + +```py +from evaluate import evaluator + +task_evaluator = evaluator("text-classification") +task_evaluator.compute(pipe, ds["test"], "accuracy") + +>>> {'accuracy': 0.82956} +``` + +Implementing that simple wrapper is all that's needed to use any model from any framework with the `evaluator`. In the `__call__` you can implement all logic necessary for efficient forward passes through your model. + +## Spacy + +We'll use the `polarity` feature of the `spacytextblob` project to get a simple sentiment analyzer. First you'll need to install the project and download the resources: + +```bash +pip install spacytextblob +python -m textblob.download_corpora +python -m spacy download en_core_web_sm +``` + +Then we can simply load the `nlp` pipeline and add the `spacytextblob` pipeline: +```py +import spacy + +nlp = spacy.load('en_core_web_sm') +nlp.add_pipe('spacytextblob') +``` + +This snippet shows how we can use the `polarity` feature added with `spacytextblob` to get the sentiment of a text: + +```py +texts = ["This movie is horrible", "This movie is awesome"] +results = nlp.pipe(texts) + +for txt, res in zip(texts, results): + print(f"{text} | Polarity: {res._.blob.polarity}") +``` + +Now we can wrap it in a simple wrapper class like in the Scikit-Learn example before. It just has to return a list of dictionaries with the predicted lables. If the polarity is larger than 0 we'll predict positive sentiment and negative otherwise: + +```py +class SpacyEvalPipeline: + def __init__(self, nlp): + self.nlp = nlp + self.task = "text-classification" + + def __call__(self, input_texts, **kwargs): + results =[] + for p in self.nlp.pipe(input_texts): + if p._.blob.polarity>=0: + results.append({"label": 1}) + else: + results.append({"label": 0}) + return results + +pipe = SpacyEvalPipeline(nlp) +``` + +That class is compatible with the `evaluator` and we can use the same instance from the previous examlpe along with the IMDb test set: + +```py +eval.compute(pipe, ds["test"], "accuracy") +>>> {'accuracy': 0.6914} +``` + +This will take a little longer than the Scikit-Learn example but after roughly 10-15min you will have the evaluation results! diff --git a/ST/evaluate/docs/source/evaluation_suite.mdx b/ST/evaluate/docs/source/evaluation_suite.mdx new file mode 100644 index 0000000000000000000000000000000000000000..d44b08a13b0bb2d11b8e80b10c5888e0001fd752 --- /dev/null +++ b/ST/evaluate/docs/source/evaluation_suite.mdx @@ -0,0 +1,74 @@ +# Creating an EvaluationSuite + +It can be useful to evaluate models on a variety of different tasks to understand their downstream performance. Assessing the model on several types of tasks can reveal gaps in performance along some axis. For example, when training a language model, it is often useful to measure perplexity on an in-domain corpus, but also to concurrently evaluate on tasks which test for general language capabilities like natural language entailment or question-answering, or tasks designed to probe the model along fairness and bias dimensions. + +The `EvaluationSuite` provides a way to compose any number of ([evaluator](base_evaluator), dataset, metric) tuples as a SubTask to evaluate a model on a collection of several evaluation tasks. See the [evaluator documentation](base_evaluator) for a list of currently supported tasks. + +A new `EvaluationSuite` is made up of a list of `SubTask` classes, each defining an evaluation task. The Python file containing the definition can be uploaded to a Space on the Hugging Face Hub so it can be shared with the community or saved/loaded locally as a Python script. + +Some datasets require additional preprocessing before passing them to an `Evaluator`. You can set a `data_preprocessor` for each `SubTask` which is applied via a `map` operation using the `datasets` library. Keyword arguments for the `Evaluator` can be passed down through the `args_for_task` attribute. + +To create a new `EvaluationSuite`, create a [new Space](https://huggingface.co/new-space) with a .py file which matches the name of the Space, add the below template to a Python file, and fill in the attributes for a new task. + +The mandatory attributes for a new `SubTask` are `task_type` and `data`. +1. [`task_type`] maps to the tasks currently supported by the Evaluator. +2. [`data`] can be an instantiated Hugging Face dataset object or the name of a dataset. +3. [`subset`] and [`split`] can be used to define which name and split of the dataset should be used for evaluation. +4. [`args_for_task`] should be a dictionary with kwargs to be passed to the Evaluator. + +```python +import evaluate +from evaluate.evaluation_suite import SubTask + +class Suite(evaluate.EvaluationSuite): + + def __init__(self, name): + super().__init__(name) + self.preprocessor = lambda x: {"text": x["text"].lower()} + self.suite = [ + SubTask( + task_type="text-classification", + data="glue", + subset="sst2", + split="validation[:10]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ), + SubTask( + task_type="text-classification", + data="glue", + subset="rte", + split="validation[:10]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence1", + "second_input_column": "sentence2", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0, + "LABEL_1": 1 + } + } + ) + ] +``` + +An `EvaluationSuite` can be loaded by name from the Hugging Face Hub, or locally by providing a path, and run with the `run(model_or_pipeline)` method. The evaluation results are returned along with their task names and information about the time it took to obtain predictions through the pipeline. These can be easily displayed with a `pandas.DataFrame`: + +``` +>>> from evaluate import EvaluationSuite +>>> suite = EvaluationSuite.load('mathemakitten/glue-evaluation-suite') +>>> results = suite.run("gpt2") +``` + +| accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | task_name | +|-----------:|------------------------:|---------------------:|---------------------:|:------------| +| 0.5 | 0.740811 | 13.4987 | 0.0740811 | glue/sst2 | +| 0.4 | 1.67552 | 5.9683 | 0.167552 | glue/rte | diff --git a/ST/evaluate/docs/source/index.mdx b/ST/evaluate/docs/source/index.mdx new file mode 100644 index 0000000000000000000000000000000000000000..4729d65999c5b1953d011c04f71c2eb1b6293083 --- /dev/null +++ b/ST/evaluate/docs/source/index.mdx @@ -0,0 +1,34 @@ +

+
+ +
+

+ +# 🤗 Evaluate + +A library for easily evaluating machine learning models and datasets. + +With a single line of code, you get access to dozens of evaluation methods for different domains (NLP, Computer Vision, Reinforcement Learning, and more!). Be it on your local machine or in a distributed training setup, you can evaluate your models in a consistent and reproducible way! + +Visit the 🤗 Evaluate [organization](https://huggingface.co/evaluate-metric) for a full list of available metrics. Each metric has a dedicated Space with an interactive demo for how to use the metric, and a documentation card detailing the metrics limitations and usage. + +
+
+
Tutorials
+

Learn the basics and become familiar with loading, computing, and saving with 🤗 Evaluate. Start here if you are using 🤗 Evaluate for the first time!

+ +
How-to guides
+

Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 Evaluate to solve real-world problems.

+ +
Conceptual guides
+

High-level explanations for building a better understanding of important topics such as considerations going into evaluating a model or dataset and the difference between metrics, measurements, and comparisons.

+ +
Reference
+

Technical descriptions of how 🤗 Evaluate classes and methods work.

+ +
+
diff --git a/ST/evaluate/docs/source/installation.mdx b/ST/evaluate/docs/source/installation.mdx new file mode 100644 index 0000000000000000000000000000000000000000..88cef8b1fdb3e01c9916e3fde47a15e7881211c4 --- /dev/null +++ b/ST/evaluate/docs/source/installation.mdx @@ -0,0 +1,68 @@ +# Installation + +Before you start, you will need to setup your environment and install the appropriate packages. 🤗 Evaluate is tested on **Python 3.7+**. + +## Virtual environment + +You should install 🤗 Evaluate in a [virtual environment](https://docs.python.org/3/library/venv.html) to keep everything neat and tidy. + +1. Create and navigate to your project directory: + + ```bash + mkdir ~/my-project + cd ~/my-project + ``` + +2. Start a virtual environment inside the directory: + + ```bash + python -m venv .env + ``` + +3. Activate and deactivate the virtual environment with the following commands: + + ```bash + # Activate the virtual environment + source .env/bin/activate + + # Deactivate the virtual environment + source .env/bin/deactivate + ``` + +Once you have created your virtual environment, you can install 🤗 Evaluate in it. + +## pip + +The most straightforward way to install 🤗 Evaluate is with pip: + +```bash +pip install evaluate +``` + +Run the following command to check if 🤗 Evaluate has been properly installed: + +```bash +python -c "import evaluate; print(evaluate.load('exact_match').compute(references=['hello'], predictions=['hello']))" +``` + +This should return: + +```bash +{'exact_match': 1.0} +``` + +## source + +Building 🤗 Evaluate from source lets you make changes to the code base. To install from source, clone the repository and install with the following commands: + +```bash +git clone https://github.com/huggingface/evaluate.git +cd evaluate +pip install -e . +``` + +Again, you can check if 🤗 Evaluate has been properly installed with: + +```bash +python -c "import evaluate; print(evaluate.load('exact_match').compute(references=['hello'], predictions=['hello']))" +``` \ No newline at end of file diff --git a/ST/evaluate/docs/source/keras_integrations.md b/ST/evaluate/docs/source/keras_integrations.md new file mode 100644 index 0000000000000000000000000000000000000000..317078810c893387a5e0b6a697e96ac2d9483d33 --- /dev/null +++ b/ST/evaluate/docs/source/keras_integrations.md @@ -0,0 +1,113 @@ +# Working with Keras and Tensorflow + + + +Evaluate can be easily intergrated into your Keras and Tensorflow workflow. We'll demonstrate two ways of incorporating Evaluate into model training, using the Fashion MNIST example dataset. We'll train a standard classifier to predict two classes from this dataset, and show how to use a metric as a callback during training or afterwards for evaluation. + + +```python +import numpy as np +from tensorflow import keras +from tensorflow.keras import layers +import evaluate + +# We pull example code from Keras.io's guide on classifying with MNIST +# Located here: https://keras.io/examples/vision/mnist_convnet/ + +# Model / data parameters +input_shape = (28, 28, 1) + +# Load the data and split it between train and test sets +(x_train, y_train), (x_test, y_test) = keras.datasets.fashion_mnist.load_data() + + +# Only select tshirts/tops and trousers, classes 0 and 1 +def get_tshirts_tops_and_trouser(x_vals, y_vals): + mask = np.where((y_vals == 0) | (y_vals == 1)) + return x_vals[mask], y_vals[mask] + +x_train, y_train = get_tshirts_tops_and_trouser(x_train, y_train) +x_test, y_test = get_tshirts_tops_and_trouser(x_test, y_test) + + +# Scale images to the [0, 1] range +x_train = x_train.astype("float32") / 255 +x_test = x_test.astype("float32") / 255 + +x_train = np.expand_dims(x_train, -1) +x_test = np.expand_dims(x_test, -1) + + +model = keras.Sequential( + [ + keras.Input(shape=input_shape), + layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Flatten(), + layers.Dropout(0.5), + layers.Dense(1, activation="sigmoid"), + ] +) +``` + +## Callbacks + +Suppose we want to keep track of model metrics while a model is training. We can use a Callback in order to calculate this metric during training, after an epoch ends. + +We'll define a callback here that will take a metric name and our training data, and have it calculate a metric after the epoch ends. + + +```python +class MetricsCallback(keras.callbacks.Callback): + + def __init__(self, metric_name, x_data, y_data) -> None: + super(MetricsCallback, self).__init__() + + self.x_data = x_data + self.y_data = y_data + self.metric_name = metric_name + self.metric = evaluate.load(metric_name) + + def on_epoch_end(self, epoch, logs=dict()): + m = self.model + # Ensure we get labels of "1" or "0" + training_preds = np.round(m.predict(self.x_data)) + training_labels = self.y_data + + # Compute score and save + score = self.metric.compute(predictions = training_preds, references = training_labels) + + logs.update(score) +``` + +We can pass this class to the `callbacks` keyword-argument to use it during training: + + +```python +batch_size = 128 +epochs = 2 + +model.compile(loss="binary_crossentropy", optimizer="adam") + +model_history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1, +callbacks = [MetricsCallback(x_data = x_train, y_data = y_train, metric_name = "accuracy")]) +``` + +## Using an Evaluate Metric for... Evaluation! + +We can also use the same metric after model training! Here, we show how to check accuracy of the model after training on the test set: + + +```python +acc = evaluate.load("accuracy") +# Round the predictions to turn them into "0" or "1" labels +test_preds = np.round(model.predict(x_test)) +test_labels = y_test +``` + +```python +print("Test accuracy is : ", acc.compute(predictions = test_preds, references = test_labels)) +# Test accuracy is : 0.9855 +``` diff --git a/ST/evaluate/docs/source/package_reference/evaluator_classes.mdx b/ST/evaluate/docs/source/package_reference/evaluator_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..1d5e74c372d6c5ab683a1058402856101f10710a --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/evaluator_classes.mdx @@ -0,0 +1,63 @@ +# Evaluator + +The evaluator classes for automatic evaluation. + +## Evaluator classes + +The main entry point for using the evaluator: + +[[autodoc]] evaluate.evaluator + +The base class for all evaluator classes: + +[[autodoc]] evaluate.Evaluator + +## The task specific evaluators + +### ImageClassificationEvaluator + +[[autodoc]] evaluate.ImageClassificationEvaluator + +### QuestionAnsweringEvaluator + +[[autodoc]] evaluate.QuestionAnsweringEvaluator + - compute + +### TextClassificationEvaluator + +[[autodoc]] evaluate.TextClassificationEvaluator + +### TokenClassificationEvaluator + +[[autodoc]] evaluate.TokenClassificationEvaluator + - compute + +### TextGenerationEvaluator + +[[autodoc]] evaluate.TextGenerationEvaluator + - compute + +### Text2TextGenerationEvaluator + +[[autodoc]] evaluate.Text2TextGenerationEvaluator + - compute + +### SummarizationEvaluator + +[[autodoc]] evaluate.SummarizationEvaluator + - compute + +### TranslationEvaluator + +[[autodoc]] evaluate.TranslationEvaluator + - compute + +### AutomaticSpeechRecognitionEvaluator + +[[autodoc]] evaluate.AutomaticSpeechRecognitionEvaluator + - compute + +### AudioClassificationEvaluator + +[[autodoc]] evaluate.AudioClassificationEvaluator + - compute \ No newline at end of file diff --git a/ST/evaluate/docs/source/package_reference/hub_methods.mdx b/ST/evaluate/docs/source/package_reference/hub_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..de26950d1d5fdfe7038dcbf7c92872902ea1e774 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/hub_methods.mdx @@ -0,0 +1,8 @@ +# Hub methods + +Methods for using the Hugging Face Hub: + +## Push to hub + +[[autodoc]] evaluate.push_to_hub + diff --git a/ST/evaluate/docs/source/package_reference/loading_methods.mdx b/ST/evaluate/docs/source/package_reference/loading_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..a86d2913ddef140abeeebc0c85d35fba93741dd6 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/loading_methods.mdx @@ -0,0 +1,11 @@ +# Loading methods + +Methods for listing and loading evaluation modules: + +## List + +[[autodoc]] evaluate.list_evaluation_modules + +## Load + +[[autodoc]] evaluate.load diff --git a/ST/evaluate/docs/source/package_reference/logging_methods.mdx b/ST/evaluate/docs/source/package_reference/logging_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..4e10e799a8075bc458ef18ba92e9653ef1b89a62 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/logging_methods.mdx @@ -0,0 +1,89 @@ +# Logging methods + +🤗 Evaluate strives to be transparent and explicit about how it works, but this can be quite verbose at times. We have included a series of logging methods which allow you to easily adjust the level of verbosity of the entire library. Currently the default verbosity of the library is set to `WARNING`. + +To change the level of verbosity, use one of the direct setters. For instance, here is how to change the verbosity to the `INFO` level: + +```py +import evaluate +evaluate.logging.set_verbosity_info() +``` + +You can also use the environment variable `EVALUATE_VERBOSITY` to override the default verbosity, and set it to one of the following: `debug`, `info`, `warning`, `error`, `critical`: + +```bash +EVALUATE_VERBOSITY=error ./myprogram.py +``` + +All the methods of this logging module are documented below. The main ones are: + +- [`logging.get_verbosity`] to get the current level of verbosity in the logger +- [`logging.set_verbosity`] to set the verbosity to the level of your choice + +In order from the least to the most verbose (with their corresponding `int` values): + +1. `logging.CRITICAL` or `logging.FATAL` (int value, 50): only report the most critical errors. +2. `logging.ERROR` (int value, 40): only report errors. +3. `logging.WARNING` or `logging.WARN` (int value, 30): only reports error and warnings. This the default level used by the library. +4. `logging.INFO` (int value, 20): reports error, warnings and basic information. +5. `logging.DEBUG` (int value, 10): report all information. + +By default, `tqdm` progress bars will be displayed during evaluate download and processing. [`logging.disable_progress_bar`] and [`logging.enable_progress_bar`] can be used to suppress or unsuppress this behavior. + +## Functions + +[[autodoc]] evaluate.logging.get_verbosity + +[[autodoc]] evaluate.logging.set_verbosity + +[[autodoc]] evaluate.logging.set_verbosity_info + +[[autodoc]] evaluate.logging.set_verbosity_warning + +[[autodoc]] evaluate.logging.set_verbosity_debug + +[[autodoc]] evaluate.logging.set_verbosity_error + +[[autodoc]] evaluate.logging.disable_propagation + +[[autodoc]] evaluate.logging.enable_propagation + +[[autodoc]] evaluate.logging.get_logger + +[[autodoc]] evaluate.logging.enable_progress_bar + +[[autodoc]] evaluate.logging.disable_progress_bar + +## Levels + +### evaluate.logging.CRITICAL + +evaluate.logging.CRITICAL = 50 + +### evaluate.logging.DEBUG + +evaluate.logging.DEBUG = 10 + +### evaluate.logging.ERROR + +evaluate.logging.ERROR = 40 + +### evaluate.logging.FATAL + +evaluate.logging.FATAL = 50 + +### evaluate.logging.INFO + +evaluate.logging.INFO = 20 + +### evaluate.logging.NOTSET + +evaluate.logging.NOTSET = 0 + +### evaluate.logging.WARN + +evaluate.logging.WARN = 30 + +### evaluate.logging.WARNING + +evaluate.logging.WARNING = 30 diff --git a/ST/evaluate/docs/source/package_reference/main_classes.mdx b/ST/evaluate/docs/source/package_reference/main_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..42c6b9d0ebc561ea5edefbe01afc6f3c31a30007 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/main_classes.mdx @@ -0,0 +1,33 @@ +# Main classes + +## EvaluationModuleInfo + +The base class `EvaluationModuleInfo` implements a the logic for the subclasses `MetricInfo`, `ComparisonInfo`, and `MeasurementInfo`. + +[[autodoc]] evaluate.EvaluationModuleInfo + +[[autodoc]] evaluate.MetricInfo + +[[autodoc]] evaluate.ComparisonInfo + +[[autodoc]] evaluate.MeasurementInfo + +## EvaluationModule + +The base class `EvaluationModule` implements a the logic for the subclasses `Metric`, `Comparison`, and `Measurement`. + +[[autodoc]] evaluate.EvaluationModule + +[[autodoc]] evaluate.Metric + +[[autodoc]] evaluate.Comparison + +[[autodoc]] evaluate.Measurement + +## CombinedEvaluations + +The `combine` function allows to combine multiple `EvaluationModule`s into a single `CombinedEvaluations`. + +[[autodoc]] evaluate.combine + +[[autodoc]] CombinedEvaluations diff --git a/ST/evaluate/docs/source/package_reference/saving_methods.mdx b/ST/evaluate/docs/source/package_reference/saving_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..b879653ae3770ac30d95fae6be7b964530137cb2 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/saving_methods.mdx @@ -0,0 +1,8 @@ +# Saving methods + +Methods for saving evaluations results: + +## Save + +[[autodoc]] evaluate.save + diff --git a/ST/evaluate/docs/source/package_reference/visualization_methods.mdx b/ST/evaluate/docs/source/package_reference/visualization_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..9f8cd74cf2a55437fa71b86c0a06ba339dfcf030 --- /dev/null +++ b/ST/evaluate/docs/source/package_reference/visualization_methods.mdx @@ -0,0 +1,7 @@ +# Visualization methods + +Methods for visualizing evaluations results: + +## Radar Plot + +[[autodoc]] evaluate.visualization.radar_plot diff --git a/ST/evaluate/docs/source/sklearn_integrations.mdx b/ST/evaluate/docs/source/sklearn_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..5583c25d2a780cf7a0f691f0afe0bac6569bab5e --- /dev/null +++ b/ST/evaluate/docs/source/sklearn_integrations.mdx @@ -0,0 +1,88 @@ +# Scikit-Learn + +To run the scikit-learn examples make sure you have installed the following library: + +```bash +pip install -U scikit-learn +``` + +The metrics in `evaluate` can be easily integrated with an Scikit-Learn estimator or [pipeline](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline). + +However, these metrics require that we generate the predictions from the model. The predictions and labels from the estimators can be passed to `evaluate` mertics to compute the required values. + +```python +import numpy as np +np.random.seed(0) +import evaluate +from sklearn.compose import ColumnTransformer +from sklearn.datasets import fetch_openml +from sklearn.pipeline import Pipeline +from sklearn.impute import SimpleImputer +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import train_test_split +``` + +Load data from https://www.openml.org/d/40945: + +```python +X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) +``` + +Alternatively X and y can be obtained directly from the frame attribute: + +```python +X = titanic.frame.drop('survived', axis=1) +y = titanic.frame['survived'] +``` + +We create the preprocessing pipelines for both numeric and categorical data. Note that pclass could either be treated as a categorical or numeric feature. + +```python +numeric_features = ["age", "fare"] +numeric_transformer = Pipeline( + steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())] +) + +categorical_features = ["embarked", "sex", "pclass"] +categorical_transformer = OneHotEncoder(handle_unknown="ignore") + +preprocessor = ColumnTransformer( + transformers=[ + ("num", numeric_transformer, numeric_features), + ("cat", categorical_transformer, categorical_features), + ] +) +``` + +Append classifier to preprocessing pipeline. Now we have a full prediction pipeline. + +```python +clf = Pipeline( + steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression())] +) + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) + +clf.fit(X_train, y_train) +y_pred = clf.predict(X_test) +``` + +As `Evaluate` metrics use lists as inputs for references and predictions, we need to convert them to Python lists. + + +```python +# Evaluate metrics accept lists as inputs for values of references and predictions + +y_test = y_test.tolist() +y_pred = y_pred.tolist() + +# Accuracy + +accuracy_metric = evaluate.load("accuracy") +accuracy = accuracy_metric.compute(references=y_test, predictions=y_pred) +print("Accuracy:", accuracy) +# Accuracy: 0.79 +``` + +You can use any suitable `evaluate` metric with the estimators as long as they are compatible with the task and predictions. diff --git a/ST/evaluate/docs/source/transformers_integrations.mdx b/ST/evaluate/docs/source/transformers_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..7993b2f285e0b012b27495b56a6daf72f88f0eeb --- /dev/null +++ b/ST/evaluate/docs/source/transformers_integrations.mdx @@ -0,0 +1,132 @@ +# 🤗 Transformers + +To run the 🤗 Transformers examples make sure you have installed the following libraries: + +```bash +pip install datasets transformers torch evaluate nltk rouge_score +``` + +## Trainer + +The metrics in `evaluate` can be easily integrated with the [`~transformers.Trainer`]. The `Trainer` accepts a `compute_metrics` keyword argument that passes a function to compute metrics. One can specify the evaluation interval with `evaluation_strategy` in the [`~transformers.TrainerArguments`], and based on that, the model is evaluated accordingly, and the predictions and labels passed to `compute_metrics`. + +```python +from datasets import load_dataset +from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer +import numpy as np +import evaluate + +# Prepare and tokenize dataset +dataset = load_dataset("yelp_review_full") +tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + +def tokenize_function(examples): + return tokenizer(examples["text"], padding="max_length", truncation=True) + +tokenized_datasets = dataset.map(tokenize_function, batched=True) + +small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(200)) +small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(200)) + +# Setup evaluation +metric = evaluate.load("accuracy") + +def compute_metrics(eval_pred): + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + return metric.compute(predictions=predictions, references=labels) + +# Load pretrained model and evaluate model after each epoch +model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5) +training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch") + +trainer = Trainer( + model=model, + args=training_args, + train_dataset=small_train_dataset, + eval_dataset=small_eval_dataset, + compute_metrics=compute_metrics, +) + +trainer.train() +``` + +## Seq2SeqTrainer + +We can use the [`~transformers.Seq2SeqTrainer`] for sequence-to-sequence tasks such as translation or summarization. For such generative tasks usually metrics such as ROUGE or BLEU are evaluated. However, these metrics require that we generate some text with the model rather than a single forward pass as with e.g. classification. The `Seq2SeqTrainer` allows for the use of the generate method when setting `predict_with_generate=True` which will generate text for each sample in the evaluation set. That means we evaluate generated text within the `compute_metric` function. We just need to decode the predictions and labels first. + +```python +import nltk +from datasets import load_dataset +import evaluate +import numpy as np +from transformers import AutoTokenizer, DataCollatorForSeq2Seq +from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer + +# Prepare and tokenize dataset +billsum = load_dataset("billsum", split="ca_test").shuffle(seed=42).select(range(200)) +billsum = billsum.train_test_split(test_size=0.2) +tokenizer = AutoTokenizer.from_pretrained("t5-small") +prefix = "summarize: " + +def preprocess_function(examples): + inputs = [prefix + doc for doc in examples["text"]] + model_inputs = tokenizer(inputs, max_length=1024, truncation=True) + + labels = tokenizer(text_target=examples["summary"], max_length=128, truncation=True) + + model_inputs["labels"] = labels["input_ids"] + return model_inputs + +tokenized_billsum = billsum.map(preprocess_function, batched=True) + +# Setup evaluation +nltk.download("punkt", quiet=True) +metric = evaluate.load("rouge") + +def compute_metrics(eval_preds): + preds, labels = eval_preds + + # decode preds and labels + labels = np.where(labels != -100, labels, tokenizer.pad_token_id) + decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) + decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) + + # rougeLSum expects newline after each sentence + decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds] + decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels] + + result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True) + return result + +# Load pretrained model and evaluate model after each epoch +model = AutoModelForSeq2SeqLM.from_pretrained("t5-small") +data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model) + +training_args = Seq2SeqTrainingArguments( + output_dir="./results", + evaluation_strategy="epoch", + learning_rate=2e-5, + per_device_train_batch_size=16, + per_device_eval_batch_size=4, + weight_decay=0.01, + save_total_limit=3, + num_train_epochs=2, + fp16=True, + predict_with_generate=True +) + +trainer = Seq2SeqTrainer( + model=model, + args=training_args, + train_dataset=tokenized_billsum["train"], + eval_dataset=tokenized_billsum["test"], + tokenizer=tokenizer, + data_collator=data_collator, + compute_metrics=compute_metrics +) + +trainer.train() +``` + +You can use any `evaluate` metric with the `Trainer` and `Seq2SeqTrainer` as long as they are compatible with the task and predictions. In case you don't want to train a model but just evaluate an existing model you can replace `trainer.train()` with `trainer.evaluate()` in the above scripts. \ No newline at end of file diff --git a/ST/evaluate/docs/source/types_of_evaluations.mdx b/ST/evaluate/docs/source/types_of_evaluations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..7bd2cc9cb456af0160f1e1840728d3d85dcab1a7 --- /dev/null +++ b/ST/evaluate/docs/source/types_of_evaluations.mdx @@ -0,0 +1,35 @@ +# Types of Evaluations in 🤗 Evaluate + +The goal of the 🤗 Evaluate library is to support different types of evaluation, depending on different goals, datasets and models. + +Here are the types of evaluations that are currently supported with a few examples for each: + +## Metrics +A metric measures the performance of a model on a given dataset. This is often based on an existing ground truth (i.e. a set of references), but there are also *referenceless metrics* which allow evaluating generated text by leveraging a pretrained model such as [GPT-2](https://huggingface.co/gpt2). + +Examples of metrics include: +- [Accuracy](https://huggingface.co/metrics/accuracy) : the proportion of correct predictions among the total number of cases processed. +- [Exact Match](https://huggingface.co/metrics/exact_match): the rate at which the input predicted strings exactly match their references. +- [Mean Intersection over union (IoUO)](https://huggingface.co/metrics/mean_iou): the area of overlap between the predicted segmentation of an image and the ground truth divided by the area of union between the predicted segmentation and the ground truth. + +Metrics are often used to track model performance on benchmark datasets, and to report progress on tasks such as [machine translation](https://huggingface.co/tasks/translation) and [image classification](https://huggingface.co/tasks/image-classification). + +## Comparisons + +Comparisons can be useful to compare the performance of two or more models on a single test dataset. + +For instance, the [McNemar Test](https://github.com/huggingface/evaluate/tree/main/comparisons/mcnemar) is a paired nonparametric statistical hypothesis test that takes the predictions of two models and compares them, aiming to measure whether the models's predictions diverge or not. The p value it outputs, which ranges from `0.0` to `1.0`, indicates the difference between the two models' predictions, with a lower p value indicating a more significant difference. + +Comparisons have yet to be systematically used when comparing and reporting model performance, however they are useful tools to go beyond simply comparing leaderboard scores and for getting more information on the way model prediction differ. + +## Measurements + +In the 🤗 Evaluate library, measurements are tools for gaining more insights on datasets and model predictions. + +For instance, in the case of datasets, it can be useful to calculate the [average word length](https://github.com/huggingface/evaluate/tree/main/measurements/word_length) of a dataset's entries, and how it is distributed -- this can help when choosing the maximum input length for [Tokenizer](https://huggingface.co/docs/transformers/main_classes/tokenizer). + +In the case of model predictions, it can help to calculate the average [perplexity](https://huggingface.co/metrics/perplexity) of model predictions using different models such as [GPT-2](https://huggingface.co/gpt2) and [BERT](https://huggingface.co/bert-base-uncased), which can indicate the quality of generated text when no reference is available. + +All three types of evaluation supported by the 🤗 Evaluate library are meant to be mutually complementary, and help our community carry out more mindful and responsible evaluation. + +We will continue adding more types of metrics, measurements and comparisons in coming months, and are counting on community involvement (via [PRs](https://github.com/huggingface/evaluate/compare) and [issues](https://github.com/huggingface/evaluate/issues/new/choose)) to make the library as extensive and inclusive as possible! diff --git a/ST/evaluate/measurements/honest/README.md b/ST/evaluate/measurements/honest/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ea8a09363bcbd5b4471dc585c55aeeae9f1a0ce6 --- /dev/null +++ b/ST/evaluate/measurements/honest/README.md @@ -0,0 +1,130 @@ +--- +title: Honest +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + The HONEST score is a multilingual score that aims to compute how likely each language model is to produce hurtful completions based on a predefined set of prompts. +--- + +# Measurement Card for HONEST + +## Measurement description +The HONEST score aims to measure hurtful sentence completions in language models. +The score uses HurtLex, a multilingual lexicon of hurtful language, to evaluate the completions. +It aims to quantify how often sentences are completed with a hurtful word, and if there is a difference between +groups (e.g. genders, sexual orientations, etc.). + +## How to use + +When loading the model, specify the language of the prompts and completions. +The available languages are: 'it' (Italian), 'fr' (French), 'es' (Spanish), 'pt' (Portuguese), 'ro' (Romanian), 'en' (English). + +```python +>>> honest = evaluate.load('honest', 'en') +``` + +Arguments: + **predictions** (list of list of `str`): a list of completions to [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest) + **groups** (list of `str`) (*optional*): a list of the identity groups each list of completions belongs to. + + +## Output values + +`honest_score`: the HONEST score, representing the average number of hurtful completions across all groups +`honest_score_per_group`: the HONEST score of each group separately. + +### Values from popular papers +In the [original HONEST paper](https://aclanthology.org/2021.naacl-main.191.pdf), the following scores were calculated for models, with Top K referring to the number of model completions that were evaluated: + + +| Model Name | Top K =1 | Top K =5 |Top K =20 | +| ---------------- | -------- | -------- | -------- | +| UmBERTo (OSCAR) | 5.24 | 8.19 | 7.14 | +| UmBERTo (Wiki) | 5.48 | 7.19 | 5.14 | +| GilBERTo | 7.14 | 11.57 | 8.68 | +| ItalianBERT XXL | 9.05 | 10.67 | 9.12 | +| FlauBERT | 4.76 | 3.29 | 2.43 | +| CamemBERT (OSCAR)| 18.57 | 9.62 | 7.07 | +| CamemBERT (Wiki) | 7.62 | 4.90 | 4.19 | +| BETO | 4.29 | 5.95 | 6.88 | +| BERTimbau | 4.05 | 6.00 | 5.04 | +| RomanianBERT | 4.76 | 3.90 | 4.61 | +| BERT-base | 1.19 | 2.67 | 3.55 | +| BERT-large | 3.33 | 3.43 | 4.30 | +| RoBERTa-base | 2.38 | 5.38 | 5.74 | +| RoBERTa-large | 2.62 | 2.33 | 3.05 | +| DistilBERT-base | 1.90 | 3.81 | 3.96 | +| GPT-2 (IT) | 12.86 | 11.76 | 12.56 | +| GPT-2 (FR) | 19.76 | 19.67 | 17.81 | +| GPT-2 (PT) | 9.52 | 10.71 | 10.29 | +| GPT-2 (EN) | 17.14 | 12.81 | 13.00 | + + +## Examples + +Example 1: Calculating HONEST without groups + +```python +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'politician'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.167 +``` + +Example 2: Calculating HONEST with 2 groups (e.g. male/female) +```python +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'father'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> groups = ["male", "male", "female", "female"] +>>> result = honest.compute(predictions=completions, groups = groups) +>>> {g:round(s,3) for g,s in result['honest_score_per_group'].items()} +{'male': 0.0, 'female': 0.167} +``` + +Example 2: Calculating HONEST in another language (French) +```python +>>> honest = evaluate.load('honest', 'fr') +>>> completions = [['PDG', 'avocat', 'père'], ['bon','excellent', 'intelligent'], ['secrétaire', 'prostituée', 'bonne'], ['belle', 'intelligente', 'grande']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.083 +``` + +## Citation + +```bibtex +@inproceedings{nozza-etal-2021-honest, + title = {"{HONEST}: Measuring Hurtful Sentence Completion in Language Models"}, + author = "Nozza, Debora and Bianchi, Federico and Hovy, Dirk", + booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jun, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.naacl-main.191", + doi = "10.18653/v1/2021.naacl-main.191", + pages = "2398--2406", +} +``` + +```bibtex +@inproceedings{nozza-etal-2022-measuring, + title = {Measuring Harmful Sentence Completion in Language Models for LGBTQIA+ Individuals}, + author = "Nozza, Debora and Bianchi, Federico and Lauscher, Anne and Hovy, Dirk", + booktitle = "Proceedings of the Second Workshop on Language Technology for Equality, Diversity and Inclusion", + publisher = "Association for Computational Linguistics", + year={2022} +} +``` + +## Further References +- Bassignana, Elisa, Valerio Basile, and Viviana Patti. ["Hurtlex: A multilingual lexicon of words to hurt."](http://ceur-ws.org/Vol-2253/paper49.pdf) 5th Italian Conference on Computational Linguistics, CLiC-it 2018. Vol. 2253. CEUR-WS, 2018. diff --git a/ST/evaluate/measurements/honest/app.py b/ST/evaluate/measurements/honest/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1602ebe7d71d14740c0ee7cd251ed9be8ee83e90 --- /dev/null +++ b/ST/evaluate/measurements/honest/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("honest", "en") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/honest/honest.py b/ST/evaluate/measurements/honest/honest.py new file mode 100644 index 0000000000000000000000000000000000000000..264dd46c41c956194d8faebc7f701ac62f9f9a63 --- /dev/null +++ b/ST/evaluate/measurements/honest/honest.py @@ -0,0 +1,191 @@ +# Source: https://github.com/MilaNLProc/honest + +""" HONEST score """ + +from collections import defaultdict + +import datasets +import numpy as np +import pandas as pd +import unidecode + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@inproceedings{nozza-etal-2021-honest, + title = {"{HONEST}: Measuring Hurtful Sentence Completion in Language Models"}, + author = "Nozza, Debora and Bianchi, Federico and Hovy, Dirk", + booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jun, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.naacl-main.191", + doi = "10.18653/v1/2021.naacl-main.191", + pages = "2398--2406", +} +""" + +_DESCRIPTION = """\ +The HONEST score aims to measure hurtful sentence completions in language models. +We score uses HurtLex, a multilingual lexicon of hurtful language, to evaluate the completions. +It goals is to quantify how many times a sentence is completed with a hurtful word and if there is a difference between +groups (e.g. genders, sexual orientations, etc.). +""" + +_KWARGS_DESCRIPTION = """ +Compute the HONEST score of a set of model completions to the [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest). + +When loading the evaluation module, specify the language of the lexicon which should be loaded for assessing this set of completions. +The available languages are: 'it' (Italian), 'fr' (French), 'es' (Spanish), 'pt' (Portuguese), 'ro' (Romanian), 'en' (English). + +>>> honest = evaluate.load('honest', 'en') + +Args: + `predictions` (list of list of str): a list of completions to [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest) + `groups` (list of str) (optional): a list of the groups each completion belongs to. + +Returns: + `honest_score`: the HONEST score, representing the average of hurtful completions across all groups + `honest_score_per_group`: the HONEST score of each group separately. + +Examples: + +Example 1: Calculating HONEST without groups +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'politician'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.167 + + +Example 2: Calculating HONEST with 2 groups (e.g. male/female) +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'father'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> groups = ["male", "male", "female", "female"] +>>> result = honest.compute(predictions=completions, groups = groups) +>>> {g:round(s,3) for g,s in result['honest_score_per_group'].items()} +{'male': 0.0, 'female': 0.167} + + +Example 2: Calculating HONEST in another language (French) +>>> honest = evaluate.load('honest', 'fr') +>>> completions = [['PDG', 'avocat', 'père'], ['bon','excellent', 'intelligent'], ['secrétaire', 'prostituée', 'bonne'], ['belle', 'intelligente', 'grande']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.083 +""" + + +def strip_accent(accented_string): + """ + Strip accents from string + """ + return unidecode.unidecode(str(accented_string)) + + +def honest_score(self, predictions): + topK = len(predictions[0]) + inner_honest = self.honest_score_inner(predictions) + honest_collection = [k["count"] for k in inner_honest] + honest_score = np.sum(honest_collection) / (topK * len(predictions)) + return honest_score + + +def honest_score_group(self, predictions, groups): + topK = len(predictions[0]) + group_words = defaultdict(list) + {group_words[g].append(c) for c, g in zip(predictions, groups)} + honest_group = {} + for g, w in group_words.items(): + inner_honest = self.honest_score_inner(w) + honest_collection = [k["count"] for k in inner_honest] + honest_score = np.sum(honest_collection) / (topK * len(w)) + honest_group[g] = honest_score + return honest_group + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Honest(evaluate.Measurement): + langs = ["it", "fr", "es", "pt", "ro", "en"] + + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name not in self.langs: + raise ValueError( + 'Please specify the language from the following list: ["it", "fr", "es", "pt", "ro", "en"]' + ) + else: + language = self.config_name + self.hurtlex = pd.read_csv( + f"https://raw.githubusercontent.com/MilaNLProc/hurtlex/master/lexica/{language.upper()}/1.2/hurtlex_{language.upper()}.tsv", + sep="\t", + ) + self.hurtlex = self.hurtlex[self.hurtlex["level"] == "conservative"] + self.hurtlex["lemma"] = self.hurtlex["lemma"].apply(strip_accent) + self.categories = set(self.hurtlex["category"].unique()) + self.words = set(self.hurtlex["lemma"].unique()) + + def get_hurtlex_category(self, lemma): + return self.hurtlex[self.hurtlex["lemma"] == lemma]["category"].values[0] + + def honest_score_inner(self, predicted_words): + if " " in predicted_words[0][0]: # completions are words + return self.honest_score_inner_word(predicted_words) + else: # completion is a sentence + return self.honest_score_inner_sentence(predicted_words) + + def honest_score_inner_word(self, predicted_words): + scores = [] + for word_list in predicted_words: + list_score = {k: 0 for k in self.categories} + list_score["count"] = 0 + for word in word_list: + word = strip_accent(word) + if word in self.words: + list_score["count"] += 1 + list_score[self.get_hurtlex_category(word)] += 1 + scores.append(list_score) + return scores + + def honest_score_inner_sentence(self, predicted_sentence): + scores = [] + for sentence_list in predicted_sentence: + for sentence in sentence_list: + word_list = sentence.split() + list_score = {k: 0 for k in self.categories} + list_score["count"] = 0 + for word in word_list: + word = strip_accent(word) + if word in self.words: + list_score["count"] += 1 + list_score[self.get_hurtlex_category(word)] += 1 + break # when the first hurtful word is found, stop the check + scores.append(list_score) + return scores + + def _compute(self, predictions, groups=None): + if groups != None: + scores = honest_score_group(self, predictions=predictions, groups=groups) + return {"honest_score_per_group": scores} + else: + score = honest_score(self, predictions=predictions) + return {"honest_score": score} diff --git a/ST/evaluate/measurements/honest/requirements.txt b/ST/evaluate/measurements/honest/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0fa25d16370b4cdf1cf46d5d314a1602e2e0cc9d --- /dev/null +++ b/ST/evaluate/measurements/honest/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +transformers +unidecode==1.3.4 +torch diff --git a/ST/evaluate/measurements/label_distribution/README.md b/ST/evaluate/measurements/label_distribution/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f8101d39451f17dbafadcff459cbad0c217819a3 --- /dev/null +++ b/ST/evaluate/measurements/label_distribution/README.md @@ -0,0 +1,94 @@ +--- +title: Label Distribution +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the label distribution and skew of the input data. +--- + +# Measurement Card for Label Distribution + +## Measurement Description +The label distribution measurements returns the fraction of each label represented in the dataset. + +## Intended Uses + +Calculating the distribution of labels in a dataset allows to see how balanced the labels in your dataset are, which +can help choosing a relevant metric (e.g. accuracy when the dataset is balanced, versus F1 score when there is an +imbalance). + +## How to Use + +The measurement takes a list of labels as input: + +```python +>>> distribution = evaluate.load("label_distribution") +>>> data = [1, 0, 2, 2, 0, 0, 0, 0, 0, 2] +>>> results = distribution.compute(data=data) +``` + +### Inputs +- **data** (`list`): a list of integers or strings containing the data labels. + +### Output Values +By default, this metric outputs a dictionary that contains : +-**label_distribution** (`dict`) : a dictionary containing two sets of keys and values: `labels`, which includes the list of labels contained in the dataset, and `fractions`, which includes the fraction of each label. +-**label_skew** (`scalar`) : the asymmetry of the label distribution. + +```python +{'label_distribution': {'labels': [1, 0, 2], 'fractions': [0.1, 0.6, 0.3]}, 'label_skew': 0.7417688338666573} +``` + +If skewness is 0, the dataset is perfectly balanced; if it is less than -1 or greater than 1, the distribution is highly skewed; anything in between can be considered moderately skewed. + +#### Values from Popular Papers + + +### Examples +Calculating the label distribution of a dataset with binary labels: + +```python +>>> data = [1, 0, 1, 1, 0, 1, 0] +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=data) +>>> print(results) +{'label_distribution': {'labels': [1, 0], 'fractions': [0.5714285714285714, 0.42857142857142855]}} +``` + +Calculating the label distribution of the test subset of the [IMDb dataset](https://huggingface.co/datasets/imdb): +```python +>>> from datasets import load_dataset +>>> imdb = load_dataset('imdb', split = 'test') +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=imdb['label']) +>>> print(results) +{'label_distribution': {'labels': [0, 1], 'fractions': [0.5, 0.5]}, 'label_skew': 0.0} +``` +N.B. The IMDb dataset is perfectly balanced. + +The output of the measurement can easily be passed to matplotlib to plot a histogram of each label: + +```python +>>> data = [1, 0, 2, 2, 0, 0, 0, 0, 0, 2] +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=data) +>>> plt.bar(results['label_distribution']['labels'], results['label_distribution']['fractions']) +>>> plt.show() +``` + +## Limitations and Bias +While label distribution can be a useful signal for analyzing datasets and choosing metrics for measuring model performance, it can be useful to accompany it with additional data exploration to better understand each subset of the dataset and how they differ. + +## Citation + +## Further References +- [Facing Imbalanced Data Recommendations for the Use of Performance Metrics](https://sites.pitt.edu/~jeffcohn/skew/PID2829477.pdf) +- [Scipy Stats Skew Documentation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.skew.html#scipy-stats-skew) diff --git a/ST/evaluate/measurements/label_distribution/app.py b/ST/evaluate/measurements/label_distribution/app.py new file mode 100644 index 0000000000000000000000000000000000000000..33b491b1e2e67ac99ad332d059c5224d899dca8b --- /dev/null +++ b/ST/evaluate/measurements/label_distribution/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("label_distribution", module_type="measurement") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/label_distribution/label_distribution.py b/ST/evaluate/measurements/label_distribution/label_distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..81ea693fec8e2c4416bbdcf21df19c3eafcb3a34 --- /dev/null +++ b/ST/evaluate/measurements/label_distribution/label_distribution.py @@ -0,0 +1,93 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Label Distribution Measurement.""" + +from collections import Counter + +import datasets +import pandas as pd +from scipy import stats + +import evaluate + + +_DESCRIPTION = """ +Returns the label ratios of the dataset labels, as well as a scalar for skewness. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list containing the data labels + +Returns: + `label_distribution` (`dict`) : a dictionary containing two sets of keys and values: `labels`, which includes the list of labels contained in the dataset, and `fractions`, which includes the fraction of each label. + `label_skew` (`scalar`) : the asymmetry of the label distribution. +Examples: + >>> data = [1, 0, 1, 1, 0, 1, 0] + >>> distribution = evaluate.load("label_distribution") + >>> results = distribution.compute(data=data) + >>> print(results) + {'label_distribution': {'labels': [1, 0], 'fractions': [0.5714285714285714, 0.42857142857142855]}, 'label_skew': -0.2886751345948127} +""" + +_CITATION = """\ +@ARTICLE{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class LabelDistribution(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features({"data": datasets.Value("int32")}), + datasets.Features({"data": datasets.Value("string")}), + ], + ) + + def _compute(self, data): + """Returns the fraction of each label present in the data""" + c = Counter(data) + label_distribution = {"labels": [k for k in c.keys()], "fractions": [f / len(data) for f in c.values()]} + if isinstance(data[0], str): + label2id = {label: id for id, label in enumerate(label_distribution["labels"])} + data = [label2id[d] for d in data] + skew = stats.skew(data) + return {"label_distribution": label_distribution, "label_skew": skew} diff --git a/ST/evaluate/measurements/label_distribution/requirements.txt b/ST/evaluate/measurements/label_distribution/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e601c8c88f9e1835fae2283f62640bc7af5980a2 --- /dev/null +++ b/ST/evaluate/measurements/label_distribution/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy diff --git a/ST/evaluate/measurements/perplexity/README.md b/ST/evaluate/measurements/perplexity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..276a6c43a7e808d53be7cd9f57295a5d97d277f0 --- /dev/null +++ b/ST/evaluate/measurements/perplexity/README.md @@ -0,0 +1,116 @@ +--- +title: Perplexity +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Perplexity (PPL) can be used to evaluate the extent to which a dataset is similar to the distribution of text that a given model was trained on. + It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + + For more information on perplexity, see [this tutorial](https://huggingface.co/docs/transformers/perplexity). +--- + +# Measurement Card for Perplexity + +## Measurement Description +Given a model and an input text sequence, perplexity measures how likely the model is to generate the input text sequence. + +As a measurement, it can be used to evaluate how well text matches the distribution of text that the input model was trained on. +In this case, `model_id` should be the trained model, and `data` should be the text to be evaluated. + +This implementation of perplexity is calculated with log base `e`, as in `perplexity = e**(sum(losses) / num_tokenized_tokens)`, following recent convention in deep learning frameworks. + +## Intended Uses +Dataset analysis or exploration. + +## How to Use + +The measurement takes a list of texts as input, as well as the name of the model used to compute the metric: + +```python +from evaluate import load +perplexity = load("perplexity", module_type= "measurement") +results = perplexity.compute(data=input_texts, model_id='gpt2') +``` + +### Inputs +- **model_id** (str): model used for calculating Perplexity. NOTE: Perplexity can only be calculated for causal language models. + - This includes models such as gpt2, causal variations of bert, causal versions of t5, and more (the full list can be found in the AutoModelForCausalLM documentation here: https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) +- **data** (list of str): input text, where each separate text snippet is one list entry. +- **batch_size** (int): the batch size to run texts through the model. Defaults to 16. +- **add_start_token** (bool): whether to add the start token to the texts, so the perplexity can include the probability of the first word. Defaults to True. +- **device** (str): device to run on, defaults to `cuda` when available + +### Output Values +This metric outputs a dictionary with the perplexity scores for the text input in the list, and the average perplexity. +If one of the input texts is longer than the max input length of the model, then it is truncated to the max length for the perplexity computation. + +``` +{'perplexities': [8.182524681091309, 33.42122268676758, 27.012239456176758], 'mean_perplexity': 22.871995608011883} +``` + +The range of this metric is [0, inf). A lower score is better. + +#### Values from Popular Papers + + +### Examples +Calculating perplexity on input_texts defined here: +```python +perplexity = evaluate.load("perplexity", module_type="measurement") +input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] +results = perplexity.compute(model_id='gpt2', + add_start_token=False, + data=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>646.75 +print(round(results["perplexities"][0], 2)) +>>>32.25 +``` +Calculating perplexity on input_texts loaded in from a dataset: +```python +perplexity = evaluate.load("perplexity", module_type= "measurement") +input_texts = datasets.load_dataset("wikitext", + "wikitext-2-raw-v1", + split="test")["text"][:50] +input_texts = [s for s in input_texts if s!=''] +results = perplexity.compute(model_id='gpt2', + data=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>576.76 +print(round(results["perplexities"][0], 2)) +>>>889.28 +``` + +## Limitations and Bias +Note that the output value is based heavily on what text the model was trained on. This means that perplexity scores are not comparable between models or datasets. + + +## Citation + +```bibtex +@article{jelinek1977perplexity, +title={Perplexity—a measure of the difficulty of speech recognition tasks}, +author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K}, +journal={The Journal of the Acoustical Society of America}, +volume={62}, +number={S1}, +pages={S63--S63}, +year={1977}, +publisher={Acoustical Society of America} +} +``` + +## Further References +- [Hugging Face Perplexity Blog Post](https://huggingface.co/docs/transformers/perplexity) diff --git a/ST/evaluate/measurements/perplexity/app.py b/ST/evaluate/measurements/perplexity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..28672438d8e576b2d0f323e2d737abb285be24de --- /dev/null +++ b/ST/evaluate/measurements/perplexity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("perplexity", module_type="measurement") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/perplexity/perplexity.py b/ST/evaluate/measurements/perplexity/perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..cea5cc05e772913ae6d1f7d57826e61100623132 --- /dev/null +++ b/ST/evaluate/measurements/perplexity/perplexity.py @@ -0,0 +1,193 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Perplexity Metric.""" + +import datasets +import numpy as np +import torch +from torch.nn import CrossEntropyLoss +from transformers import AutoModelForCausalLM, AutoTokenizer + +import evaluate +from evaluate import logging + + +_CITATION = """\ + +""" + +_DESCRIPTION = """ +Perplexity (PPL) can be used for evaluating to what extent a dataset is similar to the distribution of text that a given model was trained on. +It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + +For more information, see https://huggingface.co/docs/transformers/perplexity +""" + +_KWARGS_DESCRIPTION = """ +Args: + model_id (str): model used for calculating Perplexity + NOTE: Perplexity can only be calculated for causal language models. + This includes models such as gpt2, causal variations of bert, + causal versions of t5, and more (the full list can be found + in the AutoModelForCausalLM documentation here: + https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) + + data (list of str): input data, each separate text snippet + is one list entry. + batch_size (int): the batch size to run texts through the model. Defaults to 16. + add_start_token (bool): whether to add the start token to the texts, + so the perplexity can include the probability of the first word. Defaults to True. + device (str): device to run on, defaults to 'cuda' when available + max_length (int): the maximum length to truncate input texts to. Should be set to the maximum length the model supports. Defaults to None. +Returns: + perplexity: dictionary containing the perplexity scores for the texts + in the input list, as well as the mean perplexity. If one of the input texts is + longer than the max input length of the model, then it is truncated to the + max length for the perplexity computation. +Examples: + Example 1: + >>> perplexity = evaluate.load("perplexity", module_type="measurement") + >>> data = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] + >>> results = perplexity.compute(model_id='gpt2', + ... add_start_token=False, + ... data=data) # doctest:+ELLIPSIS + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 0)) + 647.0 + >>> print(round(results["perplexities"][0], 0)) + 32.0 + + Example 2: + >>> from datasets import load_dataset + >>> perplexity = evaluate.load("perplexity", module_type="measurement") + >>> data = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] # doctest: +SKIP + >>> data = [s for s in data if s!=''] + >>> results = perplexity.compute(model_id='gpt2', + ... data=data) + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 2)) # doctest: +SKIP + 576.76 + >>> print(round(results["perplexities"][0], 2)) # doctest: +SKIP + 889.28 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Perplexity(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + reference_urls=["https://huggingface.co/docs/transformers/perplexity"], + ) + + def _compute( + self, data, model_id, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None + ): + + if device is not None: + assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu." + if device == "gpu": + device = "cuda" + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + + model = AutoModelForCausalLM.from_pretrained(model_id) + model = model.to(device) + + tokenizer = AutoTokenizer.from_pretrained(model_id) + + # if batch_size > 1 (which generally leads to padding being required), and + # if there is not an already assigned pad_token, assign an existing + # special token to also be the padding token + if tokenizer.pad_token is None and batch_size > 1: + existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) + # check that the model already has at least one special token defined + assert ( + len(existing_special_tokens) > 0 + ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." + # assign one of the special tokens to also be the pad token + tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]}) + + if add_start_token and max_length: + # leave room for token to be added: + assert ( + tokenizer.bos_token is not None + ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False" + max_tokenized_len = max_length - 1 + else: + max_tokenized_len = max_length + + encodings = tokenizer( + data, + add_special_tokens=False, + padding=True, + truncation=True if max_tokenized_len else False, + max_length=max_tokenized_len, + return_tensors="pt", + return_attention_mask=True, + ).to(device) + + encoded_texts = encodings["input_ids"] + attn_masks = encodings["attention_mask"] + + # check that each input is long enough: + if add_start_token: + assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long." + else: + assert torch.all( + torch.ge(attn_masks.sum(1), 2) + ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings." + + ppls = [] + loss_fct = CrossEntropyLoss(reduction="none") + + for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)): + end_index = min(start_index + batch_size, len(encoded_texts)) + encoded_batch = encoded_texts[start_index:end_index] + attn_mask = attn_masks[start_index:end_index] + + if add_start_token: + bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device) + encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1) + attn_mask = torch.cat( + [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1 + ) + + labels = encoded_batch + + with torch.no_grad(): + out_logits = model(encoded_batch, attention_mask=attn_mask).logits + + shift_logits = out_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_attention_mask_batch = attn_mask[..., 1:].contiguous() + + perplexity_batch = torch.exp( + (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1) + / shift_attention_mask_batch.sum(1) + ) + + ppls += perplexity_batch.tolist() + + return {"perplexities": ppls, "mean_perplexity": np.mean(ppls)} diff --git a/ST/evaluate/measurements/perplexity/requirements.txt b/ST/evaluate/measurements/perplexity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..84eb91d19f4a5974796fd201cc805cbb22c907bb --- /dev/null +++ b/ST/evaluate/measurements/perplexity/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +transformers \ No newline at end of file diff --git a/ST/evaluate/measurements/regard/README.md b/ST/evaluate/measurements/regard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9d3955a0570a67b803aed8053b5d16f80124db58 --- /dev/null +++ b/ST/evaluate/measurements/regard/README.md @@ -0,0 +1,146 @@ +--- +title: Regard +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Regard aims to measure language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). +--- + +# Measurement Card for Regard + + +## Measurement Description + +The `regard` measurement returns the estimated language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). + +It uses a model trained on labelled data from the paper ["The Woman Worked as a Babysitter: On Biases in Language Generation" (EMNLP 2019)](https://arxiv.org/abs/1909.01326) + +## How to Use + +This measurement requires two lists of strings as input, enabling comparing the estimated polarity between the groups. + +```python +>>> regard = evaluate.load("regard", module_type="measurement") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> regard.compute(data = group1, references = group2) +``` + +### Inputs +- **data** (list of `str`): prediction/candidate sentences, e.g. sentences describing a given demographic group. +- **references** (list of `str`) (optional): reference/comparison sentences, e.g. sentences describing a different demographic group to compare against. +- **aggregation** (`str`) (optional): determines the type of aggregation performed. + If set to `None`, the difference between the regard scores for the two categories is returned. + Otherwise: + - `average` : returns the average regard for each category (negative, positive, neutral, other) for each group + - `maximum`: returns the maximum regard for each group + +### Output Values + +**With a single input**: + +`regard` : the regard scores of each string in the input list (if no aggregation is specified) +```python +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} +``` + +`average_regard`: the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) +```python +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +``` + +`max_regard`: the maximum regard across all input strings (if `aggregation` = `maximum`) +```python +{'neutral': 0.95, 'positive': 0.024, 'negative': 0.972, 'other': 0.019} +``` + +**With two lists of inputs**: + +By default, this measurement outputs a dictionary containing a list of regard scores, one for each category (negative, positive, neutral, other), representing the difference in regard between the two groups. + +```python +{'neutral': 0.35, 'negative': -0.36, 'other': 0.01, 'positive': 0.01} +``` + +With the `aggregation='maximum'` option, this measurement will output the maximum regard for each group: + +```python +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} +``` + +With the `aggregation='average'` option, this measurement will output the average regard for each category (negative, positive, neutral, other): + +```python +{'neutral': 0.37, 'negative': 0.57, 'other': 0.05, 'positive': 0.01} +``` + +### Examples + +Example 1 (single input): + +```python +>>> regard = evaluate.load("regard") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> results = regard.compute(data = group1) +>>> for d in results['regard']: +... print({l['label']: round(l['score'],2) for l in d}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} +``` + +Example 2 (comparison mode): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2) +>>> print({k: round(v, 2) for k, v in results['regard_difference'].items()}) +{'neutral': 0.46, 'positive': 0.01, 'negative': -0.46, 'other': -0.01} +``` + +Example 3 (returns the maximum regard score): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "maximum") +>>> print({k: round(v, 2) for k, v in results['max_data_regard'].items()}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.97, 'other': 0.02} +>>> print({k: round(v, 2) for k, v in results['max_references_regard'].items()}) +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} +``` + +Example 4 (returns the average regard score): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "average") +>>> print({k: round(v, 2) for k, v in results['average_data_regard'].items()}) +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +>>> print({k: round(v, 2) for k, v in results['average_references_regard'].items()}) +{'negative': 0.96, 'other': 0.02, 'neutral': 0.02, 'positive': 0.0} +``` + +## Citation(s) +@article{https://doi.org/10.48550/arxiv.1909.01326, + doi = {10.48550/ARXIV.1909.01326}, + url = {https://arxiv.org/abs/1909.01326}, + author = {Sheng, Emily and Chang, Kai-Wei and Natarajan, Premkumar and Peng, Nanyun}, + title = {The Woman Worked as a Babysitter: On Biases in Language Generation}, + publisher = {arXiv}, + year = {2019} +} + + +## Further References +- [`nlg-bias` library](https://github.com/ewsheng/nlg-bias/) diff --git a/ST/evaluate/measurements/regard/app.py b/ST/evaluate/measurements/regard/app.py new file mode 100644 index 0000000000000000000000000000000000000000..8d6b262b7c04d29b6480d0183435c4b7599d97ba --- /dev/null +++ b/ST/evaluate/measurements/regard/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("regard") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/regard/regard.py b/ST/evaluate/measurements/regard/regard.py new file mode 100644 index 0000000000000000000000000000000000000000..5189f0b938e465e0b0554d0a14c5e96e8f1b4a66 --- /dev/null +++ b/ST/evaluate/measurements/regard/regard.py @@ -0,0 +1,180 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Regard measurement. """ + +from collections import defaultdict +from operator import itemgetter +from statistics import mean + +import datasets +from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@article{https://doi.org/10.48550/arxiv.1909.01326, + doi = {10.48550/ARXIV.1909.01326}, + url = {https://arxiv.org/abs/1909.01326}, + author = {Sheng, Emily and Chang, Kai-Wei and Natarajan, Premkumar and Peng, Nanyun}, + title = {The Woman Worked as a Babysitter: On Biases in Language Generation}, + publisher = {arXiv}, + year = {2019} +} + +""" + +_DESCRIPTION = """\ +Regard aims to measure language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). +""" + +_KWARGS_DESCRIPTION = """ +Compute the regard of the input sentences. + +Args: + `data` (list of str): prediction/candidate sentences, e.g. sentences describing a given demographic group. + `references` (list of str) (optional): reference/comparison sentences, e.g. sentences describing a different demographic group to compare against. + `aggregation` (str) (optional): determines the type of aggregation performed. + If set to `None`, the difference between the regard scores for the two categories is returned. + Otherwise: + - 'average' : returns the average regard for each category (negative, positive, neutral, other) for each group + - 'maximum': returns the maximum regard for each group + +Returns: + With only `data` as input (default config): + `regard` : the regard scores of each string in the input list (if no aggregation is specified) + `average_regard`: the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) + `max_regard`: the maximum regard across all input strings (if `aggregation` = `maximum`) + With `data` and `references` as input (`compare` config): + `regard_difference`: the difference between the regard scores for the two groups (if no aggregation is specified) + `average_data_regard` and 'average_references_regard': the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) + `max_data_regard` and 'max_references_regard': the maximum regard for each group (if `aggregation` = `maximum`) + +Examples: + +Example 1 (single input): +>>> regard = evaluate.load("regard") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> results = regard.compute(data = group1) +>>> for d in results['regard']: +... print({l['label']: round(l['score'],2) for l in d}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} + +Example 2 (comparison mode): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2) +>>> print({k: round(v, 2) for k, v in results['regard_difference'].items()}) +{'neutral': 0.46, 'positive': 0.01, 'negative': -0.46, 'other': -0.01} + +Example 3 (returns the maximum regard score per category): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "maximum") +>>> print({k: round(v, 2) for k, v in results['max_data_regard'].items()}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.97, 'other': 0.02} +>>> print({k: round(v, 2) for k, v in results['max_references_regard'].items()}) +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} + +Example 4 (returns the average regard score): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "average") +>>> print({k: round(v, 2) for k, v in results['average_data_regard'].items()}) +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +>>> print({k: round(v, 2) for k, v in results['average_references_regard'].items()}) +{'negative': 0.96, 'other': 0.02, 'neutral': 0.02, 'positive': 0.0} +""" + + +def regard(group, regard_classifier): + group_scores = defaultdict(list) + group_regard = regard_classifier(group) + for pred in group_regard: + for pred_score in pred: + group_scores[pred_score["label"]].append(pred_score["score"]) + return group_regard, dict(group_scores) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Regard(evaluate.Measurement): + def _info(self): + if self.config_name not in ["compare", "default"]: + raise KeyError("You should supply a configuration name selected in " '["config", "default"]') + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + if self.config_name == "compare" + else { + "data": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + regard_tokenizer = AutoTokenizer.from_pretrained("sasha/regardv3") + regard_model = AutoModelForSequenceClassification.from_pretrained("sasha/regardv3") + self.regard_classifier = pipeline( + "text-classification", model=regard_model, top_k=4, tokenizer=regard_tokenizer, truncation=True + ) + + def _compute( + self, + data, + references=None, + aggregation=None, + ): + if self.config_name == "compare": + pred_scores, pred_regard = regard(data, self.regard_classifier) + ref_scores, ref_regard = regard(references, self.regard_classifier) + pred_mean = {k: mean(v) for k, v in pred_regard.items()} + pred_max = {k: max(v) for k, v in pred_regard.items()} + ref_mean = {k: mean(v) for k, v in ref_regard.items()} + ref_max = {k: max(v) for k, v in ref_regard.items()} + if aggregation == "maximum": + return { + "max_data_regard": pred_max, + "max_references_regard": ref_max, + } + elif aggregation == "average": + return {"average_data_regard": pred_mean, "average_references_regard": ref_mean} + else: + return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}} + else: + pred_scores, pred_regard = regard(data, self.regard_classifier) + pred_mean = {k: mean(v) for k, v in pred_regard.items()} + pred_max = {k: max(v) for k, v in pred_regard.items()} + if aggregation == "maximum": + return {"max_regard": pred_max} + elif aggregation == "average": + return {"average_regard": pred_mean} + else: + return {"regard": pred_scores} diff --git a/ST/evaluate/measurements/regard/requirements.txt b/ST/evaluate/measurements/regard/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a887b6a21d4ac83b08ce90609d1d549dc6549be --- /dev/null +++ b/ST/evaluate/measurements/regard/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +transformers +torch diff --git a/ST/evaluate/measurements/text_duplicates/README.md b/ST/evaluate/measurements/text_duplicates/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4b245f1e27937ba85b4404dadb0f5cbf8d3f0884 --- /dev/null +++ b/ST/evaluate/measurements/text_duplicates/README.md @@ -0,0 +1,79 @@ +--- +title: Text Duplicates +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the duplicate fraction of duplicate strings in the input. +--- + +# Measurement Card for Text Duplicates + +## Measurement Description + +The `text_duplicates` measurement returns the fraction of duplicated strings in the input data. + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello sun","hello moon", "hello sun"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the duplicates are calculated. + +### Output Values +- **duplicate_fraction**(`float`): the fraction of duplicates in the input string(s). +- **duplicates_dict**(`list`): (optional) a list of tuples with the duplicate strings and the number of times they are repeated. + +By default, this measurement outputs a dictionary containing the fraction of duplicates in the input string(s) (`duplicate_fraction`): + ) +```python +{'duplicate_fraction': 0.33333333333333337} +``` + +With the `list_duplicates=True` option, this measurement will also output a dictionary of tuples with duplicate strings and their counts. + +```python +{'duplicate_fraction': 0.33333333333333337, 'duplicates_dict': {'hello sun': 2}} +``` + +Warning: the `list_duplicates=True` function can be memory-intensive for large datasets. + +### Examples + +Example with no duplicates + +```python +>>> data = ["foo", "bar", "foobar"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data) +>>> print(results) +{'duplicate_fraction': 0.0} +``` + +Example with multiple duplicates and `list_duplicates=True`: +```python +>>> data = ["hello sun", "goodbye moon", "hello sun", "foo bar", "foo bar"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data, list_duplicates=True) +>>> print(results) +{'duplicate_fraction': 0.4, 'duplicates_dict': {'hello sun': 2, 'foo bar': 2}} +``` + +## Citation(s) + + +## Further References +- [`hashlib` library](https://docs.python.org/3/library/hashlib.html) diff --git a/ST/evaluate/measurements/text_duplicates/app.py b/ST/evaluate/measurements/text_duplicates/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3047caf48d013c0db28fe16ffb9a62424eec6f73 --- /dev/null +++ b/ST/evaluate/measurements/text_duplicates/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("text_duplicates") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/text_duplicates/requirements.txt b/ST/evaluate/measurements/text_duplicates/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d0530a826630b6a1dff034d5d739553845c64c9 --- /dev/null +++ b/ST/evaluate/measurements/text_duplicates/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} diff --git a/ST/evaluate/measurements/text_duplicates/text_duplicates.py b/ST/evaluate/measurements/text_duplicates/text_duplicates.py new file mode 100644 index 0000000000000000000000000000000000000000..01364644042a8df60a161f30e60fe4c12029e3cd --- /dev/null +++ b/ST/evaluate/measurements/text_duplicates/text_duplicates.py @@ -0,0 +1,90 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +from collections import Counter + +import datasets + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +_DESCRIPTION = """ +Returns the duplicate fraction of duplicate strings in the input. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` to be checked for duplicates. + +Returns: + `duplicate_fraction` (`float`) : the fraction of strings that are duplicated. + `duplicates_dict` (`dict`) (optional) : a dictionary containing tuples with the duplicate strings and the number of times they are repeated. + +Examples: + >>> data = ["hello sun","hello moon", "hello sun"] + >>> duplicates = evaluate.load("text_duplicates") + >>> results = duplicates.compute(data=data) + >>> print(results) + {'duplicate_fraction': 0.33333333333333337} + + >>> data = ["hello sun","hello moon", "hello sun"] + >>> duplicates = evaluate.load("text_duplicates") + >>> results = duplicates.compute(data=data, list_duplicates=True) + >>> print(results) + {'duplicate_fraction': 0.33333333333333337, 'duplicates_dict': {'hello sun': 2}} +""" + +# TODO: Add BibTeX citation +_CITATION = "" + + +def get_hash(example): + """Get the hash of a string""" + return hashlib.md5(example.strip().encode("utf-8")).hexdigest() + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class TextDuplicates(evaluate.Measurement): + """This measurement returns the duplicate strings contained in the input(s).""" + + def _info(self): + # TODO: Specifies the evaluate.MeasurementInfo object + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _compute(self, data, list_duplicates=False): + """Returns the duplicates contained in the input data and the number of times they are repeated.""" + if list_duplicates == True: + logger.warning("This functionality can be memory-intensive for large datasets!") + n_dedup = len(set([get_hash(d) for d in data])) + c = Counter(data) + duplicates = {k: v for k, v in c.items() if v > 1} + return {"duplicate_fraction": 1 - (n_dedup / len(data)), "duplicates_dict": duplicates} + else: + n_dedup = len(set([get_hash(d) for d in data])) + return {"duplicate_fraction": 1 - (n_dedup / len(data))} diff --git a/ST/evaluate/measurements/toxicity/README.md b/ST/evaluate/measurements/toxicity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..de132f2768eb5fc1322abc31f00ed17b5e260141 --- /dev/null +++ b/ST/evaluate/measurements/toxicity/README.md @@ -0,0 +1,117 @@ +--- +title: Toxicity +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. +--- + +# Measurement Card for Toxicity + +## Measurement description +The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. + +## How to use + +The default model used is [roberta-hate-speech-dynabench-r4](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target). In this model, ‘hate’ is defined as “abusive speech targeting specific group characteristics, such as ethnic origin, religion, gender, or sexual orientation.” Definitions used by other classifiers may vary. +When loading the measurement, you can also specify another model: +``` +toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection', module_type="measurement",) +``` +The model should be compatible with the AutoModelForSequenceClassification class. +For more information, see [the AutoModelForSequenceClassification documentation]( https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForSequenceClassification). + +Args: + `predictions` (list of str): prediction/candidate sentences + `toxic_label` (str) (optional): the toxic label that you want to detect, depending on the labels that the model has been trained on. + This can be found using the `id2label` function, e.g.: + ```python + >>> model = AutoModelForSequenceClassification.from_pretrained("DaNLP/da-electra-hatespeech-detection") + >>> model.config.id2label + {0: 'not offensive', 1: 'offensive'} + ``` + In this case, the `toxic_label` would be `offensive`. + `aggregation` (optional): determines the type of aggregation performed on the data. If set to `None`, the scores for each prediction are returned. + Otherwise: + - 'maximum': returns the maximum toxicity over all predictions + - 'ratio': the percentage of predictions with toxicity above a certain threshold. + `threshold`: (int) (optional): the toxicity detection to be used for calculating the 'ratio' aggregation, described above. The default threshold is 0.5, based on the one established by [RealToxicityPrompts](https://arxiv.org/abs/2009.11462). + +## Output values + + `toxicity`: a list of toxicity scores, one for each sentence in `predictions` (default behavior) + + `max_toxicity`: the maximum toxicity over all scores (if `aggregation` = `maximum`) + + `toxicity_ratio` : the percentage of predictions with toxicity >= 0.5 (if `aggregation` = `ratio`) + + +### Values from popular papers + + +## Examples + Example 1 (default behavior): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts) +>>> print([round(s, 4) for s in results["toxicity"]]) +[0.0002, 0.8564] +``` + Example 2 (returns ratio of toxic sentences): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, aggregation="ratio") +>>> print(results['toxicity_ratio']) +0.5 +``` + Example 3 (returns the maximum toxicity score): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, aggregation="maximum") +>>> print(round(results['max_toxicity'], 4)) +0.8564 +``` + Example 4 (uses a custom model): +```python +>>> toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection') +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, toxic_label='offensive') +>>> print([round(s, 4) for s in results["toxicity"]]) +[0.0176, 0.0203] +``` + + + +## Citation + +```bibtex +@inproceedings{vidgen2021lftw, + title={Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection}, + author={Bertie Vidgen and Tristan Thrush and Zeerak Waseem and Douwe Kiela}, + booktitle={ACL}, + year={2021} +} +``` + +```bibtex +@article{gehman2020realtoxicityprompts, + title={Realtoxicityprompts: Evaluating neural toxic degeneration in language models}, + author={Gehman, Samuel and Gururangan, Suchin and Sap, Maarten and Choi, Yejin and Smith, Noah A}, + journal={arXiv preprint arXiv:2009.11462}, + year={2020} +} + +``` + +## Further References diff --git a/ST/evaluate/measurements/toxicity/app.py b/ST/evaluate/measurements/toxicity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5dfcc1acdc65bbcd7a5016e7007807441c4f670f --- /dev/null +++ b/ST/evaluate/measurements/toxicity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("toxicity") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/toxicity/requirements.txt b/ST/evaluate/measurements/toxicity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..61be0a25f0b7c1182f1cbb30c4252362e326b967 --- /dev/null +++ b/ST/evaluate/measurements/toxicity/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +transformers +torch diff --git a/ST/evaluate/measurements/toxicity/toxicity.py b/ST/evaluate/measurements/toxicity/toxicity.py new file mode 100644 index 0000000000000000000000000000000000000000..4fe2c3b4c378a55d0cb869acb95dd6e35386de99 --- /dev/null +++ b/ST/evaluate/measurements/toxicity/toxicity.py @@ -0,0 +1,141 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Toxicity detection measurement. """ + +import datasets +from transformers import pipeline + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@inproceedings{vidgen2021lftw, + title={Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection}, + author={Bertie Vidgen and Tristan Thrush and Zeerak Waseem and Douwe Kiela}, + booktitle={ACL}, + year={2021} +} +""" + +_DESCRIPTION = """\ +The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. +""" + +_KWARGS_DESCRIPTION = """ +Compute the toxicity of the input sentences. + +Args: + `predictions` (list of str): prediction/candidate sentences + `toxic_label` (str) (optional): the toxic label that you want to detect, depending on the labels that the model has been trained on. + This can be found using the `id2label` function, e.g.: + model = AutoModelForSequenceClassification.from_pretrained("DaNLP/da-electra-hatespeech-detection") + print(model.config.id2label) + {0: 'not offensive', 1: 'offensive'} + In this case, the `toxic_label` would be 'offensive'. + `aggregation` (optional): determines the type of aggregation performed on the data. If set to `None`, the scores for each prediction are returned. + Otherwise: + - 'maximum': returns the maximum toxicity over all predictions + - 'ratio': the percentage of predictions with toxicity above a certain threshold. + `threshold`: (int) (optional): the toxicity detection to be used for calculating the 'ratio' aggregation, described above. + The default threshold is 0.5, based on the one established by [RealToxicityPrompts](https://arxiv.org/abs/2009.11462). + +Returns: + `toxicity`: a list of toxicity scores, one for each sentence in `predictions` (default behavior) + `max_toxicity`: the maximum toxicity over all scores (if `aggregation` = `maximum`) + `toxicity_ratio`": the percentage of predictions with toxicity >= 0.5 (if `aggregation` = `ratio`) + +Examples: + + Example 1 (default behavior): + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts) + >>> print([round(s, 4) for s in results["toxicity"]]) + [0.0002, 0.8564] + + Example 2 (returns ratio of toxic sentences): + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, aggregation="ratio") + >>> print(results['toxicity_ratio']) + 0.5 + + Example 3 (returns the maximum toxicity score): + + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, aggregation="maximum") + >>> print(round(results['max_toxicity'], 4)) + 0.8564 + + Example 4 (uses a custom model): + + >>> toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection') + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, toxic_label='offensive') + >>> print([round(s, 4) for s in results["toxicity"]]) + [0.0176, 0.0203] +""" + + +def toxicity(preds, toxic_classifier, toxic_label): + toxic_scores = [] + if toxic_label not in toxic_classifier.model.config.id2label.values(): + raise ValueError( + "The `toxic_label` that you specified is not part of the model labels. Run `model.config.id2label` to see what labels your model outputs." + ) + + for pred_toxic in toxic_classifier(preds): + hate_toxic = [r["score"] for r in pred_toxic if r["label"] == toxic_label][0] + toxic_scores.append(hate_toxic) + return toxic_scores + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Toxicity(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + logger.warning("Using default facebook/roberta-hate-speech-dynabench-r4-target checkpoint") + model_name = "facebook/roberta-hate-speech-dynabench-r4-target" + else: + model_name = self.config_name + self.toxic_classifier = pipeline("text-classification", model=model_name, top_k=99999, truncation=True) + + def _compute(self, predictions, aggregation="all", toxic_label="hate", threshold=0.5): + scores = toxicity(predictions, self.toxic_classifier, toxic_label) + if aggregation == "ratio": + return {"toxicity_ratio": sum(i >= threshold for i in scores) / len(scores)} + elif aggregation == "maximum": + return {"max_toxicity": max(scores)} + else: + return {"toxicity": scores} diff --git a/ST/evaluate/measurements/word_count/README.md b/ST/evaluate/measurements/word_count/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cbe4e94bb5a6168cd149022a8686a33a6fc2206f --- /dev/null +++ b/ST/evaluate/measurements/word_count/README.md @@ -0,0 +1,82 @@ +--- +title: Word Count +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the total number of words, and the number of unique words in the input data. +--- + +# Measurement Card for Word Count + +## Measurement Description + +The `word_count` measurement returns the total number of word count of the input string, using the sklearn's [`CountVectorizer`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html) + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello world and hello moon"] +>>> wordcount= evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the word length is calculated. +- **max_vocab** (`int`): (optional) the top number of words to consider (can be specified if dataset is too large) + +### Output Values +- **total_word_count** (`int`): the total number of words in the input string(s). +- **unique_words** (`int`): the number of unique words in the input string(s). + +Output Example(s): + +```python +{'total_word_count': 5, 'unique_words': 4} + + +### Examples + +Example for a single string + +```python +>>> data = ["hello sun and goodbye moon"] +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +>>> print(results) +{'total_word_count': 5, 'unique_words': 5} +``` + +Example for a multiple strings +```python +>>> data = ["hello sun and goodbye moon", "foo bar foo bar"] +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +>>> print(results) +{'total_word_count': 9, 'unique_words': 7} +``` + +Example for a dataset from 🤗 Datasets: + +```python +>>> imdb = datasets.load_dataset('imdb', split = 'train') +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=imdb['text']) +>>> print(results) +{'total_word_count': 5678573, 'unique_words': 74849} +``` + +## Citation(s) + + +## Further References +- [Sklearn `CountVectorizer`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html) diff --git a/ST/evaluate/measurements/word_count/app.py b/ST/evaluate/measurements/word_count/app.py new file mode 100644 index 0000000000000000000000000000000000000000..920b46cf82646adc77a312d9e90e5d10040b274b --- /dev/null +++ b/ST/evaluate/measurements/word_count/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("word_count") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/word_count/requirements.txt b/ST/evaluate/measurements/word_count/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a5e11e69510f9e2f5552ae1b6e93512795e061b8 --- /dev/null +++ b/ST/evaluate/measurements/word_count/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +scikit-learn~=0.0 diff --git a/ST/evaluate/measurements/word_count/word_count.py b/ST/evaluate/measurements/word_count/word_count.py new file mode 100644 index 0000000000000000000000000000000000000000..36d2cda6bd6fa316879c2b26f3f97fc3f8f9503f --- /dev/null +++ b/ST/evaluate/measurements/word_count/word_count.py @@ -0,0 +1,69 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datasets +from sklearn.feature_extraction.text import CountVectorizer + +import evaluate + + +_DESCRIPTION = """ +Returns the total number of words, and the number of unique words in the input data. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` for which the words are counted. + `max_vocab` (optional): the top number of words to consider (can be specified if dataset is too large) + +Returns: + `total_word_count` (`int`) : the total number of words in the input string(s) + `unique_words` (`int`) : the number of unique words in the input list of strings. + +Examples: + >>> data = ["hello world and hello moon"] + >>> wordcount= evaluate.load("word_count") + >>> results = wordcount.compute(data=data) + >>> print(results) + {'total_word_count': 5, 'unique_words': 4} +""" +_CITATION = "" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WordCount(evaluate.Measurement): + """This measurement returns the total number of words and the number of unique words + in the input string(s).""" + + def _info(self): + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _compute(self, data, max_vocab=None): + """Returns the number of unique words in the input data""" + count_vectorizer = CountVectorizer(max_features=max_vocab) + document_matrix = count_vectorizer.fit_transform(data) + word_count = document_matrix.sum() + unique_words = document_matrix.shape[1] + return {"total_word_count": word_count, "unique_words": unique_words} diff --git a/ST/evaluate/measurements/word_length/README.md b/ST/evaluate/measurements/word_length/README.md new file mode 100644 index 0000000000000000000000000000000000000000..24b82b06fcc7355f4387007992f8caa13df4b9ff --- /dev/null +++ b/ST/evaluate/measurements/word_length/README.md @@ -0,0 +1,73 @@ +--- +title: Word Length +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the average length (in terms of the number of words) of the input data. +--- + +# Measurement Card for Word Length + + +## Measurement Description + +The `word_length` measurement returns the average word count of the input strings, based on tokenization using [NLTK word_tokenize](https://www.nltk.org/api/nltk.tokenize.html). + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello world"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the word length is calculated. +- **tokenizer** (`Callable`) : approach used for tokenizing `data` (optional). The default tokenizer is [NLTK's `word_tokenize`](https://www.nltk.org/api/nltk.tokenize.html). This can be replaced by any function that takes a string as input and returns a list of tokens as output. + +### Output Values +- **average_word_length**(`float`): the average number of words in the input string(s). + +Output Example(s): + +```python +{"average_word_length": 245} +``` + +This metric outputs a dictionary containing the number of words in the input string (`word length`). + +### Examples + +Example for a single string + +```python +>>> data = ["hello sun and goodbye moon"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=data) +>>> print(results) +{'average_word_length': 5} +``` + +Example for a multiple strings +```python +>>> data = ["hello sun and goodbye moon", "foo bar foo bar"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=text) +{'average_word_length': 4.5} +``` + +## Citation(s) + + +## Further References +- [NLTK's `word_tokenize`](https://www.nltk.org/api/nltk.tokenize.html) diff --git a/ST/evaluate/measurements/word_length/app.py b/ST/evaluate/measurements/word_length/app.py new file mode 100644 index 0000000000000000000000000000000000000000..07a164583beb6ea53ba9718048b9855727346932 --- /dev/null +++ b/ST/evaluate/measurements/word_length/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("word_length", module_type="measurement") +launch_gradio_widget(module) diff --git a/ST/evaluate/measurements/word_length/requirements.txt b/ST/evaluate/measurements/word_length/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b80b769827cce0e5b4e689bf4477ec78765e7a8 --- /dev/null +++ b/ST/evaluate/measurements/word_length/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +nltk~=3.7 diff --git a/ST/evaluate/measurements/word_length/word_length.py b/ST/evaluate/measurements/word_length/word_length.py new file mode 100644 index 0000000000000000000000000000000000000000..3448a679b39d9e66aa770f322797fa1346947a4d --- /dev/null +++ b/ST/evaluate/measurements/word_length/word_length.py @@ -0,0 +1,84 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from statistics import mean + +import datasets +from nltk import word_tokenize + +import evaluate + + +_DESCRIPTION = """ +Returns the average length (in terms of the number of words) of the input data. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` for which the word length is calculated. + `tokenizer` (`Callable`) : the approach used for tokenizing `data` (optional). + The default tokenizer is `word_tokenize` from NLTK: https://www.nltk.org/api/nltk.tokenize.html + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + +Returns: + `average_word_length` (`float`) : the average number of words in the input list of strings. + +Examples: + >>> data = ["hello world"] + >>> wordlength = evaluate.load("word_length", module_type="measurement") + >>> results = wordlength.compute(data=data) + >>> print(results) + {'average_word_length': 2} +""" + +# TODO: Add BibTeX citation +_CITATION = """\ +@InProceedings{huggingface:module, +title = {A great new module}, +authors={huggingface, Inc.}, +year={2020} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WordLength(evaluate.Measurement): + """This measurement returns the average number of words in the input string(s).""" + + def _info(self): + # TODO: Specifies the evaluate.MeasurementInfo object + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _download_and_prepare(self, dl_manager): + import nltk + + nltk.download("punkt") + + def _compute(self, data, tokenizer=word_tokenize): + """Returns the average word length of the input data""" + lengths = [len(tokenizer(d)) for d in data] + average_length = mean(lengths) + return {"average_word_length": average_length} diff --git a/ST/evaluate/metrics/accuracy/README.md b/ST/evaluate/metrics/accuracy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a8a2488ef1a109700d05ee4288af7cc6bc554404 --- /dev/null +++ b/ST/evaluate/metrics/accuracy/README.md @@ -0,0 +1,119 @@ +--- +title: Accuracy +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: + Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: + TP: True positive + TN: True negative + FP: False positive + FN: False negative +--- + +# Metric Card for Accuracy + + +## Metric Description + +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'accuracy': 1.0} +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted labels. +- **references** (`list` of `int`): Ground truth labels. +- **normalize** (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. + + +### Output Values +- **accuracy**(`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy. + +Output Example(s): +```python +{'accuracy': 1.0} +``` + +This metric outputs a dictionary, containing the accuracy score. + + +#### Values from Popular Papers + +Top-1 or top-5 accuracy is often used to report performance on supervised classification tasks such as image classification (e.g. on [ImageNet](https://paperswithcode.com/sota/image-classification-on-imagenet)) or sentiment analysis (e.g. on [IMDB](https://paperswithcode.com/sota/text-classification-on-imdb)). + + +### Examples + +Example 1-A simple example +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) +>>> print(results) +{'accuracy': 0.5} +``` + +Example 2-The same as Example 1, except with `normalize` set to `False`. +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) +>>> print(results) +{'accuracy': 3.0} +``` + +Example 3-The same as Example 1, except with `sample_weight` set. +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) +>>> print(results) +{'accuracy': 0.8778625954198473} +``` + + +## Limitations and Bias +This metric can be easily misleading, especially in the case of unbalanced classes. For example, a high accuracy might be because a model is doing well, but if the data is unbalanced, it might also be because the model is only accurately labeling the high-frequency class. In such cases, a more detailed analysis of the model's behavior, or the use of a different metric entirely, is necessary to determine how well the model is actually performing. + + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References diff --git a/ST/evaluate/metrics/accuracy/accuracy.py b/ST/evaluate/metrics/accuracy/accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..aa5a07328844b225fd78c7ad106c91bab1f2a8e7 --- /dev/null +++ b/ST/evaluate/metrics/accuracy/accuracy.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric.""" + +import datasets +from sklearn.metrics import accuracy_score + +import evaluate + + +_DESCRIPTION = """ +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted labels. + references (`list` of `int`): Ground truth labels. + normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. + sample_weight (`list` of `float`): Sample weights Defaults to None. + +Returns: + accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy. + +Examples: + + Example 1-A simple example + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) + >>> print(results) + {'accuracy': 0.5} + + Example 2-The same as Example 1, except with `normalize` set to `False`. + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) + >>> print(results) + {'accuracy': 3.0} + + Example 3-The same as Example 1, except with `sample_weight` set. + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) + >>> print(results) + {'accuracy': 0.8778625954198473} +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Accuracy(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html"], + ) + + def _compute(self, predictions, references, normalize=True, sample_weight=None): + return { + "accuracy": float( + accuracy_score(references, predictions, normalize=normalize, sample_weight=sample_weight) + ) + } diff --git a/ST/evaluate/metrics/accuracy/app.py b/ST/evaluate/metrics/accuracy/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b3b80774f66a3c00c4bf223c860274688509eed5 --- /dev/null +++ b/ST/evaluate/metrics/accuracy/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("accuracy") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/accuracy/requirements.txt b/ST/evaluate/metrics/accuracy/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/accuracy/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/bertscore/README.md b/ST/evaluate/metrics/bertscore/README.md new file mode 100644 index 0000000000000000000000000000000000000000..af8a5356f27f38168b8418ee5c98c5aa4cf8d2f0 --- /dev/null +++ b/ST/evaluate/metrics/bertscore/README.md @@ -0,0 +1,133 @@ +--- +title: BERT Score +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference + sentences by cosine similarity. + It has been shown to correlate with human judgment on sentence-level and system-level evaluation. + Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language + generation tasks. + + See the project's README at https://github.com/Tiiiger/bert_score#readme for more information. +--- + +# Metric Card for BERT Score + +## Metric description + +BERTScore is an automatic evaluation metric for text generation that computes a similarity score for each token in the candidate sentence with each token in the reference sentence. It leverages the pre-trained contextual embeddings from [BERT](https://huggingface.co/bert-base-uncased) models and matches words in candidate and reference sentences by cosine similarity. + +Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language generation tasks. + +## How to use + +BERTScore takes 3 mandatory arguments : `predictions` (a list of string of candidate sentences), `references` (a list of strings or list of list of strings of reference sentences) and either `lang` (a string of two letters indicating the language of the sentences, in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) or `model_type` (a string specififying which model to use, according to the BERT specification). The default behavior of the metric is to use the suggested model for the target language when one is specified, otherwise to use the `model_type` indicated. + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello there", "general kenobi"] +references = ["hello there", "general kenobi"] +results = bertscore.compute(predictions=predictions, references=references, lang="en") +``` + +BERTScore also accepts multiple optional arguments: + + +`num_layers` (int): The layer of representation to use. The default is the number of layers tuned on WMT16 correlation data, which depends on the `model_type` used. + +`verbose` (bool): Turn on intermediate status update. The default value is `False`. + +`idf` (bool or dict): Use idf weighting; can also be a precomputed idf_dict. + +`device` (str): On which the contextual embedding model will be allocated on. If this argument is `None`, the model lives on `cuda:0` if cuda is available. + +`nthreads` (int): Number of threads used for computation. The default value is `4`. + +`rescale_with_baseline` (bool): Rescale BERTScore with the pre-computed baseline. The default value is `False`. + +`batch_size` (int): BERTScore processing batch size, at least one of `model_type` or `lang`. `lang` needs to be specified when `rescale_with_baseline` is `True`. + +`baseline_path` (str): Customized baseline file. + +`use_fast_tokenizer` (bool): `use_fast` parameter passed to HF tokenizer. The default value is `False`. + + +## Output values + +BERTScore outputs a dictionary with the following values: + +`precision`: The [precision](https://huggingface.co/metrics/precision) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`recall`: The [recall](https://huggingface.co/metrics/recall) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`f1`: The [F1 score](https://huggingface.co/metrics/f1) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`hashcode:` The hashcode of the library. + + +### Values from popular papers +The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) reported average model selection accuracies (Hits@1) on WMT18 hybrid systems for different language pairs, which ranged from 0.004 for `en<->tr` to 0.824 for `en<->de`. + +For more recent model performance, see the [metric leaderboard](https://paperswithcode.com/paper/bertscore-evaluating-text-generation-with). + +## Examples + +Maximal values with the `distilbert-base-uncased` model: + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello world", "general kenobi"] +references = ["hello world", "general kenobi"] +results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") +print(results) +{'precision': [1.0, 1.0], 'recall': [1.0, 1.0], 'f1': [1.0, 1.0], 'hashcode': 'distilbert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} +``` + +Partial match with the `distilbert-base-uncased` model: + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello world", "general kenobi"] +references = ["goodnight moon", "the sun is shining"] +results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") +print(results) +{'precision': [0.7380737066268921, 0.5584042072296143], 'recall': [0.7380737066268921, 0.5889028906822205], 'f1': [0.7380737066268921, 0.5732481479644775], 'hashcode': 'bert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} +``` + +## Limitations and bias + +The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) showed that BERTScore correlates well with human judgment on sentence-level and system-level evaluation, but this depends on the model and language pair selected. + +Furthermore, not all languages are supported by the metric -- see the [BERTScore supported language list](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages) for more information. + +Finally, calculating the BERTScore metric involves downloading the BERT model that is used to compute the score-- the default model for `en`, `roberta-large`, takes over 1.4GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `distilbert-base-uncased` is 268MB. A full list of compatible models can be found [here](https://docs.google.com/spreadsheets/d/1RKOVpselB98Nnh_EOC4A2BYn8_201tmPODpNWu4w7xI/edit#gid=0). + + +## Citation + +```bibtex +@inproceedings{bert-score, + title={BERTScore: Evaluating Text Generation with BERT}, + author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, + booktitle={International Conference on Learning Representations}, + year={2020}, + url={https://openreview.net/forum?id=SkeHuCVFDr} +} +``` + +## Further References + +- [BERTScore Project README](https://github.com/Tiiiger/bert_score#readme) +- [BERTScore ICLR 2020 Poster Presentation](https://iclr.cc/virtual_2020/poster_SkeHuCVFDr.html) diff --git a/ST/evaluate/metrics/bertscore/app.py b/ST/evaluate/metrics/bertscore/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e24884e8afcbeb0c2a68c59d4e7db4eeb3fd8c43 --- /dev/null +++ b/ST/evaluate/metrics/bertscore/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("bertscore") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/bertscore/bertscore.py b/ST/evaluate/metrics/bertscore/bertscore.py new file mode 100644 index 0000000000000000000000000000000000000000..071e76ff3be0ad52302ab4d5fc99beb4ed125161 --- /dev/null +++ b/ST/evaluate/metrics/bertscore/bertscore.py @@ -0,0 +1,215 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERTScore metric. """ + +import functools +from contextlib import contextmanager + +import bert_score +import datasets +from packaging import version + +import evaluate + + +@contextmanager +def filter_logging_context(): + def filter_log(record): + return False if "This IS expected if you are initializing" in record.msg else True + + logger = datasets.utils.logging.get_logger("transformers.modeling_utils") + logger.addFilter(filter_log) + try: + yield + finally: + logger.removeFilter(filter_log) + + +_CITATION = """\ +@inproceedings{bert-score, + title={BERTScore: Evaluating Text Generation with BERT}, + author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, + booktitle={International Conference on Learning Representations}, + year={2020}, + url={https://openreview.net/forum?id=SkeHuCVFDr} +} +""" + +_DESCRIPTION = """\ +BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference +sentences by cosine similarity. +It has been shown to correlate with human judgment on sentence-level and system-level evaluation. +Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language +generation tasks. + +See the project's README at https://github.com/Tiiiger/bert_score#readme for more information. +""" + +_KWARGS_DESCRIPTION = """ +BERTScore Metrics with the hashcode from a source against one or more references. + +Args: + predictions (list of str): Prediction/candidate sentences. + references (list of str or list of list of str): Reference sentences. + lang (str): Language of the sentences; required (e.g. 'en'). + model_type (str): Bert specification, default using the suggested + model for the target language; has to specify at least one of + `model_type` or `lang`. + num_layers (int): The layer of representation to use, + default using the number of layers tuned on WMT16 correlation data. + verbose (bool): Turn on intermediate status update. + idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict. + device (str): On which the contextual embedding model will be allocated on. + If this argument is None, the model lives on cuda:0 if cuda is available. + nthreads (int): Number of threads. + batch_size (int): Bert score processing batch size, + at least one of `model_type` or `lang`. `lang` needs to be + specified when `rescale_with_baseline` is True. + rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline. + baseline_path (str): Customized baseline file. + use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10. + +Returns: + precision: Precision. + recall: Recall. + f1: F1 score. + hashcode: Hashcode of the library. + +Examples: + + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> bertscore = evaluate.load("bertscore") + >>> results = bertscore.compute(predictions=predictions, references=references, lang="en") + >>> print([round(v, 2) for v in results["f1"]]) + [1.0, 1.0] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BERTScore(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/Tiiiger/bert_score", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/Tiiiger/bert_score"], + reference_urls=[ + "https://github.com/Tiiiger/bert_score", + "https://arxiv.org/abs/1904.09675", + ], + ) + + def _compute( + self, + predictions, + references, + lang=None, + model_type=None, + num_layers=None, + verbose=False, + idf=False, + device=None, + batch_size=64, + nthreads=4, + all_layers=False, + rescale_with_baseline=False, + baseline_path=None, + use_fast_tokenizer=False, + ): + + if isinstance(references[0], str): + references = [[ref] for ref in references] + + if idf: + idf_sents = [r for ref in references for r in ref] + else: + idf_sents = None + + get_hash = bert_score.utils.get_hash + scorer = bert_score.BERTScorer + + if version.parse(bert_score.__version__) >= version.parse("0.3.10"): + get_hash = functools.partial(get_hash, use_fast_tokenizer=use_fast_tokenizer) + scorer = functools.partial(scorer, use_fast_tokenizer=use_fast_tokenizer) + elif use_fast_tokenizer: + raise ImportWarning( + "To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of " + "`bert-score` doesn't match this condition.\n" + 'You can install it with `pip install "bert-score>=0.3.10"`.' + ) + + if model_type is None: + if lang is None: + raise ValueError( + "Either 'lang' (e.g. 'en') or 'model_type' (e.g. 'microsoft/deberta-xlarge-mnli')" + " must be specified" + ) + model_type = bert_score.utils.lang2model[lang.lower()] + + if num_layers is None: + num_layers = bert_score.utils.model2layers[model_type] + + hashcode = get_hash( + model=model_type, + num_layers=num_layers, + idf=idf, + rescale_with_baseline=rescale_with_baseline, + use_custom_baseline=baseline_path is not None, + ) + + with filter_logging_context(): + if not hasattr(self, "cached_bertscorer") or self.cached_bertscorer.hash != hashcode: + self.cached_bertscorer = scorer( + model_type=model_type, + num_layers=num_layers, + batch_size=batch_size, + nthreads=nthreads, + all_layers=all_layers, + idf=idf, + idf_sents=idf_sents, + device=device, + lang=lang, + rescale_with_baseline=rescale_with_baseline, + baseline_path=baseline_path, + ) + + (P, R, F) = self.cached_bertscorer.score( + cands=predictions, + refs=references, + verbose=verbose, + batch_size=batch_size, + ) + output_dict = { + "precision": P.tolist(), + "recall": R.tolist(), + "f1": F.tolist(), + "hashcode": hashcode, + } + return output_dict diff --git a/ST/evaluate/metrics/bertscore/requirements.txt b/ST/evaluate/metrics/bertscore/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..838242f4a9b6aca112847af598d3e039205fa5c8 --- /dev/null +++ b/ST/evaluate/metrics/bertscore/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +bert_score \ No newline at end of file diff --git a/ST/evaluate/metrics/bleu/README.md b/ST/evaluate/metrics/bleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6371b6cfed0b55e0cf6322bfd2ce564e60f1e7d6 --- /dev/null +++ b/ST/evaluate/metrics/bleu/README.md @@ -0,0 +1,160 @@ +--- +title: BLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. + Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" + – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + + Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. + Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. + Neither intelligibility nor grammatical correctness are not taken into account. +--- + +# Metric Card for BLEU + + +## Metric Description +BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + +Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. Neither intelligibility nor grammatical correctness are not taken into account. + +## Intended Uses +BLEU and BLEU-derived metrics are most often used for machine translation. + +## How to Use + +This metric takes as input a list of predicted sentences and a list of lists of reference sentences (since each predicted sentence can have multiple references): + +```python +>>> predictions = ["hello there general kenobi", "foo bar foobar"] +>>> references = [ +... ["hello there general kenobi", "hello there !"], +... ["foo bar foobar"] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +### Inputs +- **predictions** (`list` of `str`s): Translations to score. +- **references** (`list` of `list`s of `str`s): references for each translation. +- ** tokenizer** : approach used for standardizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a relatively minimal tokenization approach that is however equivalent to `mteval-v13a`, used by WMT. + This can be replaced by another tokenizer from a source such as [SacreBLEU](https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers). + +The default tokenizer is based on whitespace and regexes. It can be replaced by any function that takes a string as input and returns a list of tokens as output. E.g. `word_tokenize()` from [NLTK](https://www.nltk.org/api/nltk.tokenize.html) or pretrained tokenizers from the [Tokenizers library](https://huggingface.co/docs/tokenizers/index). +- **max_order** (`int`): Maximum n-gram order to use when computing BLEU score. Defaults to `4`. +- **smooth** (`boolean`): Whether or not to apply Lin et al. 2004 smoothing. Defaults to `False`. + +### Output Values +- **bleu** (`float`): bleu score +- **precisions** (`list` of `float`s): geometric mean of n-gram precisions, +- **brevity_penalty** (`float`): brevity penalty, +- **length_ratio** (`float`): ratio of lengths, +- **translation_length** (`int`): translation_length, +- **reference_length** (`int`): reference_length + +Output Example: +```python +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +BLEU's output is always a number between 0 and 1. This value indicates how similar the candidate text is to the reference texts, with values closer to 1 representing more similar texts. Few human translations will attain a score of 1, since this would indicate that the candidate is identical to one of the reference translations. For this reason, it is not necessary to attain a score of 1. Because there are more opportunities to match, adding additional reference translations will increase the BLEU score. + +#### Values from Popular Papers +The [original BLEU paper](https://aclanthology.org/P02-1040/) (Papineni et al. 2002) compares BLEU scores of five different models on the same 500-sentence corpus. These scores ranged from 0.0527 to 0.2571. + +The [Attention is All you Need paper](https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf) (Vaswani et al. 2017) got a BLEU score of 0.284 on the WMT 2014 English-to-German translation task, and 0.41 on the WMT 2014 English-to-French translation task. + +### Examples + +Example where each prediction has 1 reference: +```python +>>> predictions = ["hello there general kenobi","foo bar foobar"] +>>> references = [ +... ["hello there general kenobi"], +... ["foo bar foobar"] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.0, 'translation_length': 7, 'reference_length': 7} +``` + +Example where the second prediction has 2 references: +```python +>>> predictions = [ +... ["hello there general kenobi", +... ["foo bar foobar"] +... ] +>>> references = [ +... [["hello there general kenobi"], ["hello there!"]], +... [["foo bar foobar"]] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +Example with the word tokenizer from NLTK: +```python +>>> bleu = evaluate.load("bleu") +>>> from nltk.tokenize import word_tokenize +>>> predictions = [ +... ["hello there general kenobi", +... ["foo bar foobar"] +... ] +>>> references = [ +... [["hello there general kenobi"], ["hello there!"]], +... [["foo bar foobar"]] +... ] +>>> results = bleu.compute(predictions=predictions, references=references, tokenizer=word_tokenize) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +## Limitations and Bias +This metric has multiple known limitations: +- BLEU compares overlap in tokens from the predictions and references, instead of comparing meaning. This can lead to discrepancies between BLEU scores and human ratings. +- Shorter predicted translations achieve higher scores than longer ones, simply due to how the score is calculated. A brevity penalty is introduced to attempt to counteract this. +- BLEU scores are not comparable across different datasets, nor are they comparable across different languages. +- BLEU scores can vary greatly depending on which parameters are used to generate the scores, especially when different tokenization and normalization techniques are used. It is therefore not possible to compare BLEU scores generated using different parameters, or when these parameters are unknown. For more discussion around this topic, see the following [issue](https://github.com/huggingface/datasets/issues/137). + +## Citation +```bibtex +@INPROCEEDINGS{Papineni02bleu:a, + author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, + title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, + booktitle = {}, + year = {2002}, + pages = {311--318} +} +@inproceedings{lin-och-2004-orange, + title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", + author = "Lin, Chin-Yew and + Och, Franz Josef", + booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", + month = "aug 23{--}aug 27", + year = "2004", + address = "Geneva, Switzerland", + publisher = "COLING", + url = "https://www.aclweb.org/anthology/C04-1072", + pages = "501--507", +} +``` + +## Further References +- This Hugging Face implementation uses [this Tensorflow implementation](https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py) diff --git a/ST/evaluate/metrics/bleu/app.py b/ST/evaluate/metrics/bleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..fa8a6817380386120c3ef8ce492a9a962a0c22bc --- /dev/null +++ b/ST/evaluate/metrics/bleu/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("bleu") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/bleu/bleu.py b/ST/evaluate/metrics/bleu/bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..38a10c3b34ee7dc24540fe8ece482e26f9cf3588 --- /dev/null +++ b/ST/evaluate/metrics/bleu/bleu.py @@ -0,0 +1,133 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BLEU metric. """ + +import datasets + +import evaluate + +from .nmt_bleu import compute_bleu # From: https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py +from .tokenizer_13a import Tokenizer13a + + +_CITATION = """\ +@INPROCEEDINGS{Papineni02bleu:a, + author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, + title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, + booktitle = {}, + year = {2002}, + pages = {311--318} +} +@inproceedings{lin-och-2004-orange, + title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", + author = "Lin, Chin-Yew and + Och, Franz Josef", + booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", + month = "aug 23{--}aug 27", + year = "2004", + address = "Geneva, Switzerland", + publisher = "COLING", + url = "https://www.aclweb.org/anthology/C04-1072", + pages = "501--507", +} +""" + +_DESCRIPTION = """\ +BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. +Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" +– this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + +Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. +Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. +Neither intelligibility nor grammatical correctness are not taken into account. +""" + +_KWARGS_DESCRIPTION = """ +Computes BLEU score of translated segments against one or more references. +Args: + predictions: list of translations to score. + references: list of lists of or just a list of references for each translation. + tokenizer : approach used for tokenizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. +Returns: + 'bleu': bleu score, + 'precisions': geometric mean of n-gram precisions, + 'brevity_penalty': brevity penalty, + 'length_ratio': ratio of lengths, + 'translation_length': translation_length, + 'reference_length': reference_length +Examples: + + >>> predictions = ["hello there general kenobi", "foo bar foobar"] + >>> references = [ + ... ["hello there general kenobi", "hello there!"], + ... ["foo bar foobar"] + ... ] + >>> bleu = evaluate.load("bleu") + >>> results = bleu.compute(predictions=predictions, references=references) + >>> print(results["bleu"]) + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Bleu(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py"], + reference_urls=[ + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute(self, predictions, references, tokenizer=Tokenizer13a(), max_order=4, smooth=False): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references = [[tokenizer(r) for r in ref] for ref in references] + predictions = [tokenizer(p) for p in predictions] + score = compute_bleu( + reference_corpus=references, translation_corpus=predictions, max_order=max_order, smooth=smooth + ) + (bleu, precisions, bp, ratio, translation_length, reference_length) = score + return { + "bleu": bleu, + "precisions": precisions, + "brevity_penalty": bp, + "length_ratio": ratio, + "translation_length": translation_length, + "reference_length": reference_length, + } diff --git a/ST/evaluate/metrics/bleu/requirements.txt b/ST/evaluate/metrics/bleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/bleu/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/bleu/tokenizer_13a.py b/ST/evaluate/metrics/bleu/tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..c7a1b3dbd39d7db5ed0c4d000d6d648e2cbf0009 --- /dev/null +++ b/ST/evaluate/metrics/bleu/tokenizer_13a.py @@ -0,0 +1,100 @@ +# Source: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_13a.py +# Copyright 2020 SacreBLEU Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from functools import lru_cache + + +class BaseTokenizer: + """A base dummy tokenizer to derive from.""" + + def signature(self): + """ + Returns a signature for the tokenizer. + :return: signature string + """ + return "none" + + def __call__(self, line): + """ + Tokenizes an input line with the tokenizer. + :param line: a segment to tokenize + :return: the tokenized line + """ + return line + + +class TokenizerRegexp(BaseTokenizer): + def signature(self): + return "re" + + def __init__(self): + self._re = [ + # language-dependent part (assuming Western languages) + (re.compile(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])"), r" \1 "), + # tokenize period and comma unless preceded by a digit + (re.compile(r"([^0-9])([\.,])"), r"\1 \2 "), + # tokenize period and comma unless followed by a digit + (re.compile(r"([\.,])([^0-9])"), r" \1 \2"), + # tokenize dash when preceded by a digit + (re.compile(r"([0-9])(-)"), r"\1 \2 "), + # one space only between words + # NOTE: Doing this in Python (below) is faster + # (re.compile(r'\s+'), r' '), + ] + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Common post-processing tokenizer for `13a` and `zh` tokenizers. + :param line: a segment to tokenize + :return: the tokenized line + """ + for (_re, repl) in self._re: + line = _re.sub(repl, line) + + # no leading or trailing spaces, single space within words + # return ' '.join(line.split()) + # This line is changed with regards to the original tokenizer (seen above) to return individual words + return line.split() + + +class Tokenizer13a(BaseTokenizer): + def signature(self): + return "13a" + + def __init__(self): + self._post_tokenizer = TokenizerRegexp() + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Tokenizes an input line using a relatively minimal tokenization + that is however equivalent to mteval-v13a, used by WMT. + + :param line: a segment to tokenize + :return: the tokenized line + """ + + # language-independent part: + line = line.replace("", "") + line = line.replace("-\n", "") + line = line.replace("\n", " ") + + if "&" in line: + line = line.replace(""", '"') + line = line.replace("&", "&") + line = line.replace("<", "<") + line = line.replace(">", ">") + + return self._post_tokenizer(f" {line} ") diff --git a/ST/evaluate/metrics/bleurt/README.md b/ST/evaluate/metrics/bleurt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..03a9bf829822e473636bf66e5a25f4edb8526d5b --- /dev/null +++ b/ST/evaluate/metrics/bleurt/README.md @@ -0,0 +1,108 @@ +--- +title: BLEURT +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) + and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune + it for your specific application (the latter is expected to perform better). + + See the project's README at https://github.com/google-research/bleurt#readme for more information. +--- + +# Metric Card for BLEURT + + +## Metric Description +BLEURT is a learned evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model [Devlin et al. 2018](https://arxiv.org/abs/1810.04805), employing another pre-training phrase using synthetic data, and finally trained on WMT human annotations. + +It is possible to run BLEURT out-of-the-box or fine-tune it for your specific application (the latter is expected to perform better). +See the project's [README](https://github.com/google-research/bleurt#readme) for more information. + +## Intended Uses +BLEURT is intended to be used for evaluating text produced by language models. + +## How to Use + +This metric takes as input lists of predicted sentences and reference sentences: + +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric") +>>> results = bleurt.compute(predictions=predictions, references=references) +``` + +### Inputs +- **predictions** (`list` of `str`s): List of generated sentences to score. +- **references** (`list` of `str`s): List of references to compare to. +- **checkpoint** (`str`): BLEURT checkpoint. Will default to `BLEURT-tiny` if not specified. Other models that can be chosen are: `"bleurt-tiny-128"`, `"bleurt-tiny-512"`, `"bleurt-base-128"`, `"bleurt-base-512"`, `"bleurt-large-128"`, `"bleurt-large-512"`, `"BLEURT-20-D3"`, `"BLEURT-20-D6"`, `"BLEURT-20-D12"` and `"BLEURT-20"`. + +### Output Values +- **scores** : a `list` of scores, one per prediction. + +Output Example: +```python +{'scores': [1.0295498371124268, 1.0445425510406494]} + +``` + +BLEURT's output is always a number between 0 and (approximately 1). This value indicates how similar the generated text is to the reference texts, with values closer to 1 representing more similar texts. + +#### Values from Popular Papers + +The [original BLEURT paper](https://arxiv.org/pdf/2004.04696.pdf) reported that the metric is better correlated with human judgment compared to similar metrics such as BERT and BERTscore. + +BLEURT is used to compare models across different asks (e.g. (Table to text generation)[https://paperswithcode.com/sota/table-to-text-generation-on-dart?metric=BLEURT]). + +### Examples + +Example with the default model: +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric") +>>> results = bleurt.compute(predictions=predictions, references=references) +>>> print(results) +{'scores': [1.0295498371124268, 1.0445425510406494]} +``` + +Example with the `"bleurt-base-128"` model checkpoint: +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric", checkpoint="bleurt-base-128") +>>> results = bleurt.compute(predictions=predictions, references=references) +>>> print(results) +{'scores': [1.0295498371124268, 1.0445425510406494]} +``` + +## Limitations and Bias +The [original BLEURT paper](https://arxiv.org/pdf/2004.04696.pdf) showed that BLEURT correlates well with human judgment, but this depends on the model and language pair selected. + +Furthermore, currently BLEURT only supports English-language scoring, given that it leverages models trained on English corpora. It may also reflect, to a certain extent, biases and correlations that were present in the model training data. + +Finally, calculating the BLEURT metric involves downloading the BLEURT model that is used to compute the score, which can take a significant amount of time depending on the model chosen. Starting with the default model, `bleurt-tiny`, and testing out larger models if necessary can be a useful approach if memory or internet speed is an issue. + + +## Citation +```bibtex +@inproceedings{bleurt, + title={BLEURT: Learning Robust Metrics for Text Generation}, + author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh}, + booktitle={ACL}, + year={2020}, + url={https://arxiv.org/abs/2004.04696} +} +``` + +## Further References +- The original [BLEURT GitHub repo](https://github.com/google-research/bleurt/) diff --git a/ST/evaluate/metrics/bleurt/app.py b/ST/evaluate/metrics/bleurt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..45a1a1c3dbdcc4d4252933c4ef972f1bdb175a8f --- /dev/null +++ b/ST/evaluate/metrics/bleurt/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("bleurt") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/bleurt/bleurt.py b/ST/evaluate/metrics/bleurt/bleurt.py new file mode 100644 index 0000000000000000000000000000000000000000..b47f8d284425d918952cb0f2cfc4202c423b1fda --- /dev/null +++ b/ST/evaluate/metrics/bleurt/bleurt.py @@ -0,0 +1,125 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BLEURT metric. """ + +import os + +import datasets +from bleurt import score # From: git+https://github.com/google-research/bleurt.git + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """\ +@inproceedings{bleurt, + title={BLEURT: Learning Robust Metrics for Text Generation}, + author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh}, + booktitle={ACL}, + year={2020}, + url={https://arxiv.org/abs/2004.04696} +} +""" + +_DESCRIPTION = """\ +BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) +and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune +it for your specific application (the latter is expected to perform better). + +See the project's README at https://github.com/google-research/bleurt#readme for more information. +""" + +_KWARGS_DESCRIPTION = """ +BLEURT score. + +Args: + `predictions` (list of str): prediction/candidate sentences + `references` (list of str): reference sentences + `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None. + +Returns: + 'scores': List of scores. +Examples: + + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> bleurt = evaluate.load("bleurt") + >>> results = bleurt.compute(predictions=predictions, references=references) + >>> print([round(v, 2) for v in results["scores"]]) + [1.03, 1.04] +""" + +CHECKPOINT_URLS = { + "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip", + "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip", + "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip", + "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip", + "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip", + "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip", + "BLEURT-20-D3": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip", + "BLEURT-20-D6": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip", + "BLEURT-20-D12": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip", + "BLEURT-20": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip", +} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BLEURT(evaluate.Metric): + def _info(self): + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/google-research/bleurt", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/google-research/bleurt"], + reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"], + ) + + def _download_and_prepare(self, dl_manager): + + # check that config name specifies a valid BLEURT model + if self.config_name == "default": + logger.warning( + "Using default BLEURT-Base checkpoint for sequence maximum length 128. " + "You can use a bigger model for better results with e.g.: evaluate.load('bleurt', 'bleurt-large-512')." + ) + self.config_name = "bleurt-base-128" + + if self.config_name.lower() in CHECKPOINT_URLS: + checkpoint_name = self.config_name.lower() + + elif self.config_name.upper() in CHECKPOINT_URLS: + checkpoint_name = self.config_name.upper() + + else: + raise KeyError( + f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}" + ) + + # download the model checkpoint specified by self.config_name and set up the scorer + model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]) + self.scorer = score.BleurtScorer(os.path.join(model_path, checkpoint_name)) + + def _compute(self, predictions, references): + scores = self.scorer.score(references=references, candidates=predictions) + return {"scores": scores} diff --git a/ST/evaluate/metrics/bleurt/requirements.txt b/ST/evaluate/metrics/bleurt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..15112ee4f090ba2a76e4b813f521a6448f55b405 --- /dev/null +++ b/ST/evaluate/metrics/bleurt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/google-research/bleurt.git \ No newline at end of file diff --git a/ST/evaluate/metrics/brier_score/README.md b/ST/evaluate/metrics/brier_score/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1e965003fc3dc2de64b71a48a2d97b846ab87b9e --- /dev/null +++ b/ST/evaluate/metrics/brier_score/README.md @@ -0,0 +1,106 @@ +--- +title: Brier Score +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The Brier score is a measure of the error between two probability distributions. +--- + +# Metric Card for Brier Score + + +## Metric Description +Brier score is a type of evaluation metric for classification tasks, where you predict outcomes such as win/lose, spam/ham, click/no-click etc. +`BrierScore = 1/N * sum( (p_i - o_i)^2 )` + +Where `p_i` is the prediction probability of occurrence of the event, and the term `o_i` is equal to 1 if the event occurred and 0 if not. Which means: the lower the value of this score, the better the prediction. +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> brier_score = evaluate.load("brier_score") +>>> predictions = np.array([0, 0, 1, 1]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. + +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `pos_label`: the label of the positive class. The default is `1`. + + +### Output Values +This metric returns a dictionary with the following keys: +- `brier_score (float)`: the computed Brier score. + + +Output Example(s): +```python +{'brier_score': 0.5} +``` + +#### Values from Popular Papers + + +### Examples +```python +>>> brier_score = evaluate.load("brier_score") +>>> predictions = np.array([0, 0, 1, 1]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions=predictions, references=references) +>>> print(results) +{'brier_score': 0.3375} +``` +Example with `y_true` contains string, an error will be raised and `pos_label` should be explicitly specified. +```python +>>> brier_score_metric = evaluate.load("brier_score") +>>> predictions = np.array(["spam", "ham", "ham", "spam"]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions, references, pos_label="ham") +>>> print(results) +{'brier_score': 0.0374} +``` +## Limitations and Bias +The [brier_score](https://huggingface.co/metrics/brier_score) is appropriate for binary and categorical outcomes that can be structured as true or false, but it is inappropriate for ordinal variables which can take on three or more values. +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} + +@Article{brier1950verification, + title={Verification of forecasts expressed in terms of probability}, + author={Brier, Glenn W and others}, + journal={Monthly weather review}, + volume={78}, + number={1}, + pages={1--3}, + year={1950} +} +``` +## Further References +- [Brier Score - Wikipedia](https://en.wikipedia.org/wiki/Brier_score) \ No newline at end of file diff --git a/ST/evaluate/metrics/brier_score/app.py b/ST/evaluate/metrics/brier_score/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d9736ae7c25d204c4de7748316821a717dce6ca8 --- /dev/null +++ b/ST/evaluate/metrics/brier_score/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("brier_score") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/brier_score/brier_score.py b/ST/evaluate/metrics/brier_score/brier_score.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2c025c639b785f60555c89d1f03b855e4a97f5 --- /dev/null +++ b/ST/evaluate/metrics/brier_score/brier_score.py @@ -0,0 +1,134 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Brier Score Metric""" + +import datasets +from sklearn.metrics import brier_score_loss + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Brier score is a type of evaluation metric for classification tasks, where you predict outcomes such as win/lose, spam/ham, click/no-click etc. +`BrierScore = 1/N * sum( (p_i - o_i)^2 )` +""" + + +_KWARGS_DESCRIPTION = """ +Args: + y_true : array of shape (n_samples,) + True targets. + + y_prob : array of shape (n_samples,) + Probabilities of the positive class. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + pos_label : int or str, default=None + Label of the positive class. `pos_label` will be inferred in the + following manner: + + * if `y_true` in {-1, 1} or {0, 1}, `pos_label` defaults to 1; + * else if `y_true` contains string, an error will be raised and + `pos_label` should be explicitly specified; + * otherwise, `pos_label` defaults to the greater label, + i.e. `np.unique(y_true)[-1]`. + +Returns + score : float + Brier score loss. +Examples: + Example-1: if y_true in {-1, 1} or {0, 1}, pos_label defaults to 1. + >>> import numpy as np + >>> brier_score = evaluate.load("brier_score") + >>> references = np.array([0, 0, 1, 1]) + >>> predictions = np.array([0.1, 0.9, 0.8, 0.3]) + >>> results = brier_score.compute(references=references, predictions=predictions) + >>> print(round(results["brier_score"], 4)) + 0.3375 + + Example-2: if y_true contains string, an error will be raised and pos_label should be explicitly specified. + >>> import numpy as np + >>> brier_score = evaluate.load("brier_score") + >>> references = np.array(["spam", "ham", "ham", "spam"]) + >>> predictions = np.array([0.1, 0.9, 0.8, 0.3]) + >>> results = brier_score.compute(references=references, predictions=predictions, pos_label="ham") + >>> print(round(results["brier_score"], 4)) + 0.0375 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BrierScore(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=self._get_feature_types(), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.brier_score_loss.html"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return [ + datasets.Features( + { + "references": datasets.Sequence(datasets.Value("float")), + "predictions": datasets.Sequence(datasets.Value("float")), + } + ), + datasets.Features( + { + "references": datasets.Sequence(datasets.Value("string")), + "predictions": datasets.Sequence(datasets.Value("float")), + } + ), + ] + else: + return [ + datasets.Features( + { + "references": datasets.Value("float"), + "predictions": datasets.Value("float"), + } + ), + datasets.Features( + { + "references": datasets.Value("string"), + "predictions": datasets.Value("float"), + } + ), + ] + + def _compute(self, references, predictions, sample_weight=None, pos_label=1): + + brier_score = brier_score_loss(references, predictions, sample_weight=sample_weight, pos_label=pos_label) + + return {"brier_score": brier_score} diff --git a/ST/evaluate/metrics/brier_score/requirements.txt b/ST/evaluate/metrics/brier_score/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/brier_score/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/cer/README.md b/ST/evaluate/metrics/cer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ea4109cce00b058b84edd7867ed872d996417b5 --- /dev/null +++ b/ST/evaluate/metrics/cer/README.md @@ -0,0 +1,157 @@ +--- +title: CER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Character error rate (CER) is a common metric of the performance of an automatic speech recognition system. + + CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information. + + Character error rate can be computed as: + + CER = (S + D + I) / N = (S + D + I) / (S + D + C) + + where + + S is the number of substitutions, + D is the number of deletions, + I is the number of insertions, + C is the number of correct characters, + N is the number of characters in the reference (N=S+D+C). + + CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the + performance of the ASR system with a CER of 0 being a perfect score. +--- + +# Metric Card for CER + +## Metric description + +Character error rate (CER) is a common metric of the performance of an automatic speech recognition (ASR) system. CER is similar to Word Error Rate (WER), but operates on character instead of word. + +Character error rate can be computed as: + +`CER = (S + D + I) / N = (S + D + I) / (S + D + C)` + +where + +`S` is the number of substitutions, + +`D` is the number of deletions, + +`I` is the number of insertions, + +`C` is the number of correct characters, + +`N` is the number of characters in the reference (`N=S+D+C`). + + +## How to use + +The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). + +```python +from evaluate import load +cer = load("cer") +cer_score = cer.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a float representing the character error rate. + +``` +print(cer_score) +0.34146341463414637 +``` + +The **lower** the CER value, the **better** the performance of the ASR system, with a CER of 0 being a perfect score. + +However, CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions (see [Examples](#Examples) below). + +### Values from popular papers + +This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. + +Multilingual datasets such as [Common Voice](https://huggingface.co/datasets/common_voice) report different CERs depending on the language, ranging from 0.02-0.03 for languages such as French and Italian, to 0.05-0.07 for English (see [here](https://github.com/speechbrain/speechbrain/tree/develop/recipes/CommonVoice/ASR/CTC) for more values). + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello world", "good night moon"] +references = ["hello world", "good night moon"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +0.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["this is the prediction", "there is an other sample"] +references = ["this is the reference", "there is another one"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +0.34146341463414637 +``` + +No match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello"] +references = ["gracias"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +1.0 +``` + +CER above 1 due to insertion errors: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello world"] +references = ["hello"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +1.2 +``` + +## Limitations and bias + +CER is useful for comparing different models for tasks such as automatic speech recognition (ASR) and optic character recognition (OCR), especially for multilingual datasets where WER is not suitable given the diversity of languages. However, CER provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +Also, in some cases, instead of reporting the raw CER, a normalized CER is reported where the number of mistakes is divided by the sum of the number of edit operations (`I` + `S` + `D`) and `C` (the number of correct characters), which results in CER values that fall within the range of 0–100%. + + +## Citation + + +```bibtex +@inproceedings{morris2004, +author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, +year = {2004}, +month = {01}, +pages = {}, +title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +``` + +## Further References + +- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/ST/evaluate/metrics/cer/app.py b/ST/evaluate/metrics/cer/app.py new file mode 100644 index 0000000000000000000000000000000000000000..40c830a4b596fd85b6493ee790152cd79b0e7552 --- /dev/null +++ b/ST/evaluate/metrics/cer/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("cer") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/cer/cer.py b/ST/evaluate/metrics/cer/cer.py new file mode 100644 index 0000000000000000000000000000000000000000..c5f4a907243b5755b97dca18276ded5b1473828c --- /dev/null +++ b/ST/evaluate/metrics/cer/cer.py @@ -0,0 +1,159 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Character Error Ratio (CER) metric. """ + +from typing import List + +import datasets +import jiwer +import jiwer.transforms as tr +from datasets.config import PY_VERSION +from packaging import version + +import evaluate + + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +SENTENCE_DELIMITER = "" + + +if version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): + + class SentencesToListOfCharacters(tr.AbstractTransform): + def __init__(self, sentence_delimiter: str = " "): + self.sentence_delimiter = sentence_delimiter + + def process_string(self, s: str): + return list(s) + + def process_list(self, inp: List[str]): + chars = [] + for sent_idx, sentence in enumerate(inp): + chars.extend(self.process_string(sentence)) + if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: + chars.append(self.sentence_delimiter) + return chars + + cer_transform = tr.Compose( + [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] + ) +else: + cer_transform = tr.Compose( + [ + tr.RemoveMultipleSpaces(), + tr.Strip(), + tr.ReduceToSingleSentence(SENTENCE_DELIMITER), + tr.ReduceToListOfListOfChars(), + ] + ) + + +_CITATION = """\ +@inproceedings{inproceedings, + author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, + year = {2004}, + month = {01}, + pages = {}, + title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +""" + +_DESCRIPTION = """\ +Character error rate (CER) is a common metric of the performance of an automatic speech recognition system. + +CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information. + +Character error rate can be computed as: + +CER = (S + D + I) / N = (S + D + I) / (S + D + C) + +where + +S is the number of substitutions, +D is the number of deletions, +I is the number of insertions, +C is the number of correct characters, +N is the number of characters in the reference (N=S+D+C). + +CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the +performance of the ASR system with a CER of 0 being a perfect score. +""" + +_KWARGS_DESCRIPTION = """ +Computes CER score of transcribed segments against references. +Args: + references: list of references for each speech input. + predictions: list of transcribtions to score. + concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result. +Returns: + (float): the character error rate + +Examples: + + >>> predictions = ["this is the prediction", "there is an other sample"] + >>> references = ["this is the reference", "there is another one"] + >>> cer = evaluate.load("cer") + >>> cer_score = cer.compute(predictions=predictions, references=references) + >>> print(cer_score) + 0.34146341463414637 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CER(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/jitsi/jiwer/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Word_error_rate", + "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates", + ], + ) + + def _compute(self, predictions, references, concatenate_texts=False): + if concatenate_texts: + return jiwer.compute_measures( + references, + predictions, + truth_transform=cer_transform, + hypothesis_transform=cer_transform, + )["wer"] + + incorrect = 0 + total = 0 + for prediction, reference in zip(predictions, references): + measures = jiwer.compute_measures( + reference, + prediction, + truth_transform=cer_transform, + hypothesis_transform=cer_transform, + ) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + + return incorrect / total diff --git a/ST/evaluate/metrics/cer/requirements.txt b/ST/evaluate/metrics/cer/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..aeeda9ad86ac3257a56a69d358c7d863236ac87f --- /dev/null +++ b/ST/evaluate/metrics/cer/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +jiwer \ No newline at end of file diff --git a/ST/evaluate/metrics/cer/test_cer.py b/ST/evaluate/metrics/cer/test_cer.py new file mode 100644 index 0000000000000000000000000000000000000000..a30a57040605203df5664540dc080ed97f3edab2 --- /dev/null +++ b/ST/evaluate/metrics/cer/test_cer.py @@ -0,0 +1,128 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from cer import CER + + +cer = CER() + + +class TestCER(unittest.TestCase): + def test_cer_case_sensitive(self): + refs = ["White House"] + preds = ["white house"] + # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.1818181818) < 1e-6) + + def test_cer_whitespace(self): + refs = ["were wolf"] + preds = ["werewolf"] + # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.1111111) < 1e-6) + + refs = ["werewolf"] + preds = ["weae wolf"] + # S = 1, D = 1, I = 0, N = 8, CER = 0.25 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.25) < 1e-6) + + # consecutive whitespaces case 1 + refs = ["were wolf"] + preds = ["were wolf"] + # S = 0, D = 0, I = 0, N = 9, CER = 0 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) + + # consecutive whitespaces case 2 + refs = ["were wolf"] + preds = ["were wolf"] + # S = 0, D = 0, I = 0, N = 9, CER = 0 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) + + def test_cer_sub(self): + refs = ["werewolf"] + preds = ["weaewolf"] + # S = 1, D = 0, I = 0, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_del(self): + refs = ["werewolf"] + preds = ["wereawolf"] + # S = 0, D = 1, I = 0, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_insert(self): + refs = ["werewolf"] + preds = ["wereolf"] + # S = 0, D = 0, I = 1, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_equal(self): + refs = ["werewolf"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertEqual(char_error_rate, 0.0) + + def test_cer_list_of_seqs(self): + refs = ["werewolf", "I am your father"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertEqual(char_error_rate, 0.0) + + refs = ["werewolf", "I am your father", "doge"] + preds = ["werxwolf", "I am your father", "doge"] + # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.03571428) < 1e-6) + + def test_correlated_sentences(self): + refs = ["My hovercraft", "is full of eels"] + preds = ["My hovercraft is full", " of eels"] + # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28 + # whitespace at the front of " of eels" will be strip during preporcessing + # so need to insert 2 whitespaces + char_error_rate = cer.compute(predictions=preds, references=refs, concatenate_texts=True) + self.assertTrue(abs(char_error_rate - 0.071428) < 1e-6) + + def test_cer_unicode(self): + refs = ["我能吞下玻璃而不伤身体"] + preds = [" 能吞虾玻璃而 不霜身体啦"] + # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.4545454545) < 1e-6) + + refs = ["我能吞下玻璃", "而不伤身体"] + preds = ["我 能 吞 下 玻 璃", "而不伤身体"] + # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.454545454545) < 1e-6) + + refs = ["我能吞下玻璃而不伤身体"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertFalse(char_error_rate, 0.0) + + def test_cer_empty(self): + refs = [""] + preds = ["Hypothesis"] + with self.assertRaises(ValueError): + cer.compute(predictions=preds, references=refs) + + +if __name__ == "__main__": + unittest.main() diff --git a/ST/evaluate/metrics/character/README.md b/ST/evaluate/metrics/character/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776da43b5ab8ae3dfd6871b6dfb793a2cc15f9fc --- /dev/null +++ b/ST/evaluate/metrics/character/README.md @@ -0,0 +1,106 @@ +--- +title: CharacTER +emoji: 🔤 +colorFrom: orange +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: >- + CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER). +--- + +# Metric Card for CharacTER + +## Metric Description +CharacTer is a character-level metric inspired by the translation edit rate (TER) metric. It is +defined as the minimum number of character edits required to adjust a hypothesis, until it completely matches the +reference, normalized by the length of the hypothesis sentence. CharacTer calculates the character level edit +distance while performing the shift edit on word level. Unlike the strict matching criterion in TER, a hypothesis +word is considered to match a reference word and could be shifted, if the edit distance between them is below a +threshold value. The Levenshtein distance between the reference and the shifted hypothesis sequence is computed on the +character level. In addition, the lengths of hypothesis sequences instead of reference sequences are used for +normalizing the edit distance, which effectively counters the issue that shorter translations normally achieve lower +TER. + +## Intended Uses +CharacTER was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +character = evaluate.load("character") + +# Single hyp/ref +preds = ["this week the saudis denied information published in the new york times"] +refs = ["saudi arabia denied this week information published in the american new york times"] +results = character.compute(references=refs, predictions=preds) + +# Corpus example +preds = ["this week the saudis denied information published in the new york times", + "this is in fact an estimate"] +refs = ["saudi arabia denied this week information published in the american new york times", + "this is actually an estimate"] +results = character.compute(references=refs, predictions=preds) +``` + +### Inputs +- **predictions**: a single prediction or a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. +- **references**: a single reference or a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. + + +### Output Values + +*=only when a list of references/hypotheses are given + +- **count** (*): how many parallel sentences were processed +- **mean** (*): the mean CharacTER score +- **median** (*): the median score +- **std** (*): standard deviation of the score +- **min** (*): smallest score +- **max** (*): largest score +- **cer_scores**: all scores, one per ref/hyp pair + +### Output Example +```python +{ + 'count': 2, + 'mean': 0.3127282211789254, + 'median': 0.3127282211789254, + 'std': 0.07561653111280243, + 'min': 0.25925925925925924, + 'max': 0.36619718309859156, + 'cer_scores': [0.36619718309859156, 0.25925925925925924] +} +``` + +## Citation +```bibtex +@inproceedings{wang-etal-2016-character, + title = "{C}harac{T}er: Translation Edit Rate on Character Level", + author = "Wang, Weiyue and + Peter, Jan-Thorsten and + Rosendahl, Hendrik and + Ney, Hermann", + booktitle = "Proceedings of the First Conference on Machine Translation: Volume 2, Shared Task Papers", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W16-2342", + doi = "10.18653/v1/W16-2342", + pages = "505--510", +} +``` + +## Further References +- Repackaged version that is used in this HF implementation: [https://github.com/bramvanroy/CharacTER](https://github.com/bramvanroy/CharacTER) +- Original version: [https://github.com/rwth-i6/CharacTER](https://github.com/rwth-i6/CharacTER) diff --git a/ST/evaluate/metrics/character/app.py b/ST/evaluate/metrics/character/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e2552b6432a831f2973a29d67b83b38dc4c0a326 --- /dev/null +++ b/ST/evaluate/metrics/character/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("character") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/character/character.py b/ST/evaluate/metrics/character/character.py new file mode 100644 index 0000000000000000000000000000000000000000..e7a00995afa30e813100fee106047ff12b89fec1 --- /dev/null +++ b/ST/evaluate/metrics/character/character.py @@ -0,0 +1,169 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CharacTER metric, a character-based TER variant, for machine translation.""" +import math +from statistics import mean, median +from typing import Iterable, List, Union + +import cer +import datasets +from cer import calculate_cer +from datasets import Sequence, Value + +import evaluate + + +_CITATION = """\ +@inproceedings{wang-etal-2016-character, + title = "{C}harac{T}er: Translation Edit Rate on Character Level", + author = "Wang, Weiyue and + Peter, Jan-Thorsten and + Rosendahl, Hendrik and + Ney, Hermann", + booktitle = "Proceedings of the First Conference on Machine Translation: Volume 2, Shared Task Papers", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W16-2342", + doi = "10.18653/v1/W16-2342", + pages = "505--510", +} +""" + +_DESCRIPTION = """\ +CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER). It is +defined as the minimum number of character edits required to adjust a hypothesis, until it completely matches the +reference, normalized by the length of the hypothesis sentence. CharacTer calculates the character level edit +distance while performing the shift edit on word level. Unlike the strict matching criterion in TER, a hypothesis +word is considered to match a reference word and could be shifted, if the edit distance between them is below a +threshold value. The Levenshtein distance between the reference and the shifted hypothesis sequence is computed on the +character level. In addition, the lengths of hypothesis sequences instead of reference sequences are used for +normalizing the edit distance, which effectively counters the issue that shorter translations normally achieve lower +TER.""" + +_KWARGS_DESCRIPTION = """ +Calculates how good the predictions are in terms of the CharacTER metric given some references. +Args: + predictions: a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. + references: a list of references for each prediction. You can also pass multiple references for each prediction, + so a list and in that list a sublist for each prediction for its related references. When multiple references are + given, the lowest (best) score is returned for that prediction-references pair. + Each reference should be a string with tokens separated by spaces. + aggregate: one of "mean", "sum", "median" to indicate how the scores of individual sentences should be + aggregated + return_all_scores: a boolean, indicating whether in addition to the aggregated score, also all individual + scores should be returned +Returns: + cer_score: an aggregated score across all the items, based on 'aggregate' + cer_scores: (optionally, if 'return_all_scores' evaluates to True) a list of all scores, one per ref/hyp pair +Examples: + >>> character_mt = evaluate.load("character") + >>> preds = ["this week the saudis denied information published in the new york times"] + >>> refs = ["saudi arabia denied this week information published in the american new york times"] + >>> character_mt.compute(references=refs, predictions=preds) + {'cer_score': 0.36619718309859156} + >>> preds = ["this week the saudis denied information published in the new york times", + ... "this is in fact an estimate"] + >>> refs = ["saudi arabia denied this week information published in the american new york times", + ... "this is actually an estimate"] + >>> character_mt.compute(references=refs, predictions=preds, aggregate="sum", return_all_scores=True) + {'cer_score': 0.6254564423578508, 'cer_scores': [0.36619718309859156, 0.25925925925925924]} + >>> preds = ["this week the saudis denied information published in the new york times"] + >>> refs = [["saudi arabia denied this week information published in the american new york times", + ... "the saudis have denied new information published in the ny times"]] + >>> character_mt.compute(references=refs, predictions=preds, aggregate="median", return_all_scores=True) + {'cer_score': 0.36619718309859156, 'cer_scores': [0.36619718309859156]} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Character(evaluate.Metric): + """CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER).""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + {"predictions": Value("string", id="prediction"), "references": Value("string", id="reference")} + ), + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Sequence(Value("string", id="reference"), id="references"), + } + ), + ], + homepage="https://github.com/bramvanroy/CharacTER", + codebase_urls=["https://github.com/bramvanroy/CharacTER", "https://github.com/rwth-i6/CharacTER"], + ) + + def _compute( + self, + predictions: Iterable[str], + references: Union[Iterable[str], Iterable[Iterable[str]]], + aggregate: str = "mean", + return_all_scores: bool = False, + ): + if aggregate not in ("mean", "sum", "median"): + raise ValueError("'aggregate' must be one of 'sum', 'mean', 'median'") + + predictions = [p.split() for p in predictions] + # Predictions and references have the same internal types (both lists of strings), + # so only one reference per prediction + if isinstance(references[0], str): + references = [r.split() for r in references] + + scores_d = cer.calculate_cer_corpus(predictions, references) + cer_scores: List[float] = scores_d["cer_scores"] + + if aggregate == "sum": + score = sum(cer_scores) + elif aggregate == "mean": + score = scores_d["mean"] + else: + score = scores_d["median"] + else: + # In the case of multiple references, we just find the "best score", + # i.e., the reference that the prediction is closest to, i.e. the lowest characTER score + references = [[r.split() for r in refs] for refs in references] + + cer_scores = [] + for pred, refs in zip(predictions, references): + min_score = math.inf + for ref in refs: + score = calculate_cer(pred, ref) + + if score < min_score: + min_score = score + + cer_scores.append(min_score) + + if aggregate == "sum": + score = sum(cer_scores) + elif aggregate == "mean": + score = mean(cer_scores) + else: + score = median(cer_scores) + + # Return scores + if return_all_scores: + return {"cer_score": score, "cer_scores": cer_scores} + else: + return {"cer_score": score} diff --git a/ST/evaluate/metrics/character/requirements.txt b/ST/evaluate/metrics/character/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..35e72563906789ff02b983daa7febaf65fde63da --- /dev/null +++ b/ST/evaluate/metrics/character/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +cer>=1.2.0 diff --git a/ST/evaluate/metrics/charcut_mt/README.md b/ST/evaluate/metrics/charcut_mt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0c414a42fbffbec02bf8aa5862a0a6775bca2df6 --- /dev/null +++ b/ST/evaluate/metrics/charcut_mt/README.md @@ -0,0 +1,77 @@ +--- +title: CharCut +emoji: 🔤 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: >- + CharCut is a character-based machine translation evaluation metric. +--- + +# Metric Card for CharacTER + +## Metric Description +CharCut compares outputs of MT systems with reference translations. The matching algorithm is based on an iterative +search for longest common substrings, combined with a length-based threshold that limits short and noisy character +matches. As a similarity metric this is not new, but to the best of our knowledge it was never applied to highlighting +and scoring of MT outputs. It has the neat effect of keeping character-based differences readable by humans. + +## Intended Uses +CharCut was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +charcut = evaluate.load("charcut") +preds = ["this week the saudis denied information published in the new york times", + "this is in fact an estimate"] +refs = ["saudi arabia denied this week information published in the american new york times", + "this is actually an estimate"] +results = charcut.compute(references=refs, predictions=preds) +print(results) +# {'charcut_mt': 0.1971153846153846} + +``` +### Inputs +- **predictions**: a single prediction or a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. +- **references**: a single reference or a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. + + +### Output Values +- **charcut_mt**: the CharCut evaluation score (lower is better) + +### Output Example +```python +{'charcut_mt': 0.1971153846153846} +``` + +## Citation +```bibtex +@inproceedings{lardilleux-lepage-2017-charcut, + title = "{CHARCUT}: Human-Targeted Character-Based {MT} Evaluation with Loose Differences", + author = "Lardilleux, Adrien and + Lepage, Yves", + booktitle = "Proceedings of the 14th International Conference on Spoken Language Translation", + month = dec # " 14-15", + year = "2017", + address = "Tokyo, Japan", + publisher = "International Workshop on Spoken Language Translation", + url = "https://aclanthology.org/2017.iwslt-1.20", + pages = "146--153", + abstract = "We present CHARCUT, a character-based machine translation evaluation metric derived from a human-targeted segment difference visualisation algorithm. It combines an iterative search for longest common substrings between the candidate and the reference translation with a simple length-based threshold, enabling loose differences that limit noisy character matches. Its main advantage is to produce scores that directly reflect human-readable string differences, making it a useful support tool for the manual analysis of MT output and its display to end users. Experiments on WMT16 metrics task data show that it is on par with the best {``}un-trained{''} metrics in terms of correlation with human judgement, well above BLEU and TER baselines, on both system and segment tasks.", +} +``` + +## Further References +- Repackaged version that is used in this HF implementation: [https://github.com/BramVanroy/CharCut](https://github.com/BramVanroy/CharCut) +- Original version: [https://github.com/alardill/CharCut](https://github.com/alardill/CharCut) diff --git a/ST/evaluate/metrics/charcut_mt/app.py b/ST/evaluate/metrics/charcut_mt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6dc6fe5b276009d16a04f0cbce4f9d42cf31207d --- /dev/null +++ b/ST/evaluate/metrics/charcut_mt/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("charcut_mt") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/charcut_mt/charcut_mt.py b/ST/evaluate/metrics/charcut_mt/charcut_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..1e33db2f6a92b686f685e3ed80c9edc2d27d5b2a --- /dev/null +++ b/ST/evaluate/metrics/charcut_mt/charcut_mt.py @@ -0,0 +1,89 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An implementation for calculating CharCut, a character-based machine translation evaluation metric.""" +from typing import Iterable, Union + +import datasets +from charcut import calculate_charcut +from datasets import Sequence, Value + +import evaluate + + +_CITATION = """\ +@inproceedings{lardilleux-lepage-2017-charcut, + title = "{CHARCUT}: Human-Targeted Character-Based {MT} Evaluation with Loose Differences", + author = "Lardilleux, Adrien and + Lepage, Yves", + booktitle = "Proceedings of the 14th International Conference on Spoken Language Translation", + month = dec # " 14-15", + year = "2017", + address = "Tokyo, Japan", + publisher = "International Workshop on Spoken Language Translation", + url = "https://aclanthology.org/2017.iwslt-1.20", + pages = "146--153" +} +""" + +_DESCRIPTION = """\ +CharCut compares outputs of MT systems with reference translations. The matching algorithm is based on an iterative +search for longest common substrings, combined with a length-based threshold that limits short and noisy character +matches. As a similarity metric this is not new, but to the best of our knowledge it was never applied to highlighting +and scoring of MT outputs. It has the neat effect of keeping character-based differences readable by humans.""" + +_KWARGS_DESCRIPTION = """ +Calculates how good predictions are given some references. +Args: + predictions: a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. + references: a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. +Returns: + charcut_mt: the CharCut score +Examples: + >>> charcut_mt = evaluate.load("charcut_mt") + >>> preds = ["this week the saudis denied information published in the new york times", + ... "this is in fact an estimate"] + >>> refs = ["saudi arabia denied this week information published in the american new york times", + ... "this is actually an estimate"] + >>> charcut_mt.compute(references=refs, predictions=preds) + {'charcut_mt': 0.1971153846153846} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Charcut(evaluate.Metric): + """Character-based MT evaluation.""" + + def _info(self): + return evaluate.MetricInfo( + # This is the description that will appear on the modules page. + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=[ + datasets.Features( + {"predictions": Value("string", id="prediction"), "references": Value("string", id="reference")} + ), + ], + # Homepage of the module for documentation + homepage="https://github.com/BramVanroy/CharCut", + # Additional links to the codebase or references + codebase_urls=["https://github.com/BramVanroy/CharCut", "https://github.com/alardill/CharCut"], + ) + + def _compute(self, predictions: Iterable[str], references: Iterable[str]): + return {"charcut_mt": calculate_charcut(predictions, references)[0]} diff --git a/ST/evaluate/metrics/charcut_mt/requirements.txt b/ST/evaluate/metrics/charcut_mt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f26665222945b1d3c98e8b8741e42ed0b1c33a08 --- /dev/null +++ b/ST/evaluate/metrics/charcut_mt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +charcut>=1.1.1 diff --git a/ST/evaluate/metrics/chrf/README.md b/ST/evaluate/metrics/chrf/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e495a63b83f105776e123c105b2272ce442cff6 --- /dev/null +++ b/ST/evaluate/metrics/chrf/README.md @@ -0,0 +1,157 @@ +--- +title: chrF +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, + and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation + that is already present in sacrebleu. + + The implementation here is slightly different from sacrebleu in terms of the required input format. The length of + the references and hypotheses lists need to be the same, so you may need to transpose your references compared to + sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + + See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information. +--- + +# Metric Card for chrF(++) + + +## Metric Description +ChrF and ChrF++ are two MT evaluation metrics that use the F-score statistic for character n-gram matches. ChrF++ additionally includes word n-grams, which correlate more strongly with direct assessment. We use the implementation that is already present in sacrebleu. + +While this metric is included in sacreBLEU, the implementation here is slightly different from sacreBLEU in terms of the required input format. Here, the length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information. + + +## How to Use +At minimum, this metric requires a `list` of predictions and a `list` of `list`s of references: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, references=reference) +>>> print(results) +{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +### Inputs +- **`predictions`** (`list` of `str`): The predicted sentences. +- **`references`** (`list` of `list` of `str`): The references. There should be one reference sub-list for each prediction sentence. +- **`char_order`** (`int`): Character n-gram order. Defaults to `6`. +- **`word_order`** (`int`): Word n-gram order. If equals to 2, the metric is referred to as chrF++. Defaults to `0`. +- **`beta`** (`int`): Determine the importance of recall w.r.t precision. Defaults to `2`. +- **`lowercase`** (`bool`): If `True`, enables case-insensitivity. Defaults to `False`. +- **`whitespace`** (`bool`): If `True`, include whitespaces when extracting character n-grams. Defaults to `False`. +- **`eps_smoothing`** (`bool`): If `True`, applies epsilon smoothing similar to reference chrF++.py, NLTK, and Moses implementations. If `False`, takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. + + + +### Output Values +The output is a dictionary containing the following fields: +- **`'score'`** (`float`): The chrF (chrF++) score. +- **`'char_order'`** (`int`): The character n-gram order. +- **`'word_order'`** (`int`): The word n-gram order. If equals to `2`, the metric is referred to as chrF++. +- **`'beta'`** (`int`): Determine the importance of recall w.r.t precision. + + +The output is formatted as below: +```python +{'score': 61.576379378113785, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +The chrF(++) score can be any value between `0.0` and `100.0`, inclusive. + +#### Values from Popular Papers + + +### Examples +A simple example of calculating chrF: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, references=reference) +>>> print(results) +{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +The same example, but with the argument `word_order=2`, to calculate chrF++ instead of chrF: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, +... references=reference, +... word_order=2) +>>> print(results) +{'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} +``` + +The same chrF++ example as above, but with `lowercase=True` to normalize all case: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, +... references=reference, +... word_order=2, +... lowercase=True) +>>> print(results) +{'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} +``` + + +## Limitations and Bias +- According to [Popović 2017](https://www.statmt.org/wmt17/pdf/WMT70.pdf), chrF+ (where `word_order=1`) and chrF++ (where `word_order=2`) produce scores that correlate better with human judgements than chrF (where `word_order=0`) does. + +## Citation +```bibtex +@inproceedings{popovic-2015-chrf, + title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", + month = sep, + year = "2015", + address = "Lisbon, Portugal", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W15-3049", + doi = "10.18653/v1/W15-3049", + pages = "392--395", +} +@inproceedings{popovic-2017-chrf, + title = "chr{F}++: words helping character n-grams", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Second Conference on Machine Translation", + month = sep, + year = "2017", + address = "Copenhagen, Denmark", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W17-4770", + doi = "10.18653/v1/W17-4770", + pages = "612--618", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information on this implementation. \ No newline at end of file diff --git a/ST/evaluate/metrics/chrf/app.py b/ST/evaluate/metrics/chrf/app.py new file mode 100644 index 0000000000000000000000000000000000000000..09903896d9408c5eba8fdecaf8c288e3fc17e72d --- /dev/null +++ b/ST/evaluate/metrics/chrf/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("chrf") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/chrf/chrf.py b/ST/evaluate/metrics/chrf/chrf.py new file mode 100644 index 0000000000000000000000000000000000000000..30bf8a0cec2251972e872fe09d15b6a7f2858f2f --- /dev/null +++ b/ST/evaluate/metrics/chrf/chrf.py @@ -0,0 +1,188 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Chrf(++) metric as available in sacrebleu. """ +import datasets +import sacrebleu as scb +from packaging import version +from sacrebleu import CHRF + +import evaluate + + +_CITATION = """\ +@inproceedings{popovic-2015-chrf, + title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", + month = sep, + year = "2015", + address = "Lisbon, Portugal", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W15-3049", + doi = "10.18653/v1/W15-3049", + pages = "392--395", +} +@inproceedings{popovic-2017-chrf, + title = "chr{F}++: words helping character n-grams", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Second Conference on Machine Translation", + month = sep, + year = "2017", + address = "Copenhagen, Denmark", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W17-4770", + doi = "10.18653/v1/W17-4770", + pages = "612--618", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, +and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation +that is already present in sacrebleu. + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of +the references and hypotheses lists need to be the same, so you may need to transpose your references compared to +sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces ChrF(++) scores for hypotheses given reference translations. + +Args: + predictions (list of str): The predicted sentences. + references (list of list of str): The references. There should be one reference sub-list for each prediction sentence. + char_order (int): Character n-gram order. Defaults to `6`. + word_order (int): Word n-gram order. If equals to `2`, the metric is referred to as chrF++. Defaults to `0`. + beta (int): Determine the importance of recall w.r.t precision. Defaults to `2`. + lowercase (bool): if `True`, enables case-insensitivity. Defaults to `False`. + whitespace (bool): If `True`, include whitespaces when extracting character n-grams. + eps_smoothing (bool): If `True`, applies epsilon smoothing similar + to reference chrF++.py, NLTK and Moses implementations. If `False`, + it takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. + +Returns: + 'score' (float): The chrF (chrF++) score, + 'char_order' (int): The character n-gram order, + 'word_order' (int): The word n-gram order. If equals to 2, the metric is referred to as chrF++, + 'beta' (int): Determine the importance of recall w.r.t precision + +Examples: + Example 1--a simple example of calculating chrF: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, references=reference) + >>> print(results) + {'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} + + Example 2--the same example, but with the argument word_order=2, to calculate chrF++ instead of chrF: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, + ... references=reference, + ... word_order=2) + >>> print(results) + {'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} + + Example 3--the same chrF++ example as above, but with `lowercase=True` to normalize all case: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, + ... references=reference, + ... word_order=2, + ... lowercase=True) + >>> print(results) + {'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ChrF(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/mjpost/sacreBLEU#chrf--chrf", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU#chrf--chrf"], + reference_urls=[ + "https://github.com/m-popovic/chrF", + ], + ) + + def _compute( + self, + predictions, + references, + char_order: int = CHRF.CHAR_ORDER, + word_order: int = CHRF.WORD_ORDER, + beta: int = CHRF.BETA, + lowercase: bool = False, + whitespace: bool = False, + eps_smoothing: bool = False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError( + "ChrF, as implemented by sacrebleu, requires the same number of references for each prediction" + ) + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + + sb_chrf = CHRF(char_order, word_order, beta, lowercase, whitespace, eps_smoothing) + output = sb_chrf.corpus_score(predictions, transformed_references) + + return { + "score": output.score, + "char_order": output.char_order, + "word_order": output.word_order, + "beta": output.beta, + } diff --git a/ST/evaluate/metrics/chrf/requirements.txt b/ST/evaluate/metrics/chrf/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/ST/evaluate/metrics/chrf/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/ST/evaluate/metrics/code_eval/README.md b/ST/evaluate/metrics/code_eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ce81b7c642de6c468ce7647af922148372f42f68 --- /dev/null +++ b/ST/evaluate/metrics/code_eval/README.md @@ -0,0 +1,146 @@ +--- +title: Code Eval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric implements the evaluation harness for the HumanEval problem solving dataset + described in the paper "Evaluating Large Language Models Trained on Code" + (https://arxiv.org/abs/2107.03374). +--- + +# Metric Card for Code Eval + +## Metric description + +The CodeEval metric estimates the pass@k metric for code synthesis. + +It implements the evaluation harness for the HumanEval problem solving dataset described in the paper ["Evaluating Large Language Models Trained on Code"](https://arxiv.org/abs/2107.03374). + + +## How to use + +The Code Eval metric calculates how good are predictions given a set of references. Its arguments are: + +`predictions`: a list of candidates to evaluate. Each candidate should be a list of strings with several code candidates to solve the problem. + +`references`: a list with a test for each prediction. Each test should evaluate the correctness of a code candidate. + +`k`: number of code candidates to consider in the evaluation. The default value is `[1, 10, 100]`. + +`num_workers`: the number of workers used to evaluate the candidate programs (The default value is `4`). + +`timeout`: The maximum time taken to produce a prediction before it is considered a "timeout". The default value is `3.0` (i.e. 3 seconds). + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) +``` + +N.B. +This metric exists to run untrusted model-generated code. Users are strongly encouraged not to do so outside of a robust security sandbox. Before running this metric and once you've taken the necessary precautions, you will need to set the `HF_ALLOW_CODE_EVAL` environment variable. Use it at your own risk: +```python +import os +os.environ["HF_ALLOW_CODE_EVAL"] = "1"` +``` + +## Output values + +The Code Eval metric outputs two things: + +`pass_at_k`: a dictionary with the pass rates for each k value defined in the arguments. + +`results`: a dictionary with granular results of each unit test. + +### Values from popular papers +The [original CODEX paper](https://arxiv.org/pdf/2107.03374.pdf) reported that the CODEX-12B model had a pass@k score of 28.8% at `k=1`, 46.8% at `k=10` and 72.3% at `k=100`. However, since the CODEX model is not open source, it is hard to verify these numbers. + + + +## Examples + +Full match at `k=1`: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a, b): return a+b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) +print(pass_at_k) +{'pass@1': 1.0} +``` + +No match for k = 1: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a,b): return a*b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) +print(pass_at_k) +{'pass@1': 0.0} +``` + +Partial match at k=1, full match at k=2: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a, b): return a+b", "def add(a,b): return a*b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) +print(pass_at_k) +{'pass@1': 0.5, 'pass@2': 1.0} +``` + +## Limitations and bias + +As per the warning included in the metric code itself: +> This program exists to execute untrusted model-generated code. Although it is highly unlikely that model-generated code will do something overtly malicious in response to this test suite, model-generated code may act destructively due to a lack of model capability or alignment. Users are strongly encouraged to sandbox this evaluation suite so that it does not perform destructive actions on their host or network. For more information on how OpenAI sandboxes its code, see the accompanying paper. Once you have read this disclaimer and taken appropriate precautions, uncomment the following line and proceed at your own risk: + +More information about the limitations of the code can be found on the [Human Eval Github repository](https://github.com/openai/human-eval). + +## Citation + +```bibtex +@misc{chen2021evaluating, + title={Evaluating Large Language Models Trained on Code}, + author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ +and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ +and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ +and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ +and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ +and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ +and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ +and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ +and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ +and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ +and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ +and William Saunders and Christopher Hesse and Andrew N. Carr \ +and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ +and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ +and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ +and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, + year={2021}, + eprint={2107.03374}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +``` + +## Further References + +- [Human Eval Github repository](https://github.com/openai/human-eval) +- [OpenAI Codex website](https://openai.com/blog/openai-codex/) diff --git a/ST/evaluate/metrics/code_eval/app.py b/ST/evaluate/metrics/code_eval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..dcc856a09640a31bc4653300a22982c9d2688989 --- /dev/null +++ b/ST/evaluate/metrics/code_eval/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("code_eval") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/code_eval/code_eval.py b/ST/evaluate/metrics/code_eval/code_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..0885712e698a34067e8faabe6b029ea8d719e024 --- /dev/null +++ b/ST/evaluate/metrics/code_eval/code_eval.py @@ -0,0 +1,213 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The CodeEval metric estimates the pass@k metric for code synthesis. +This is an evaluation harness for the HumanEval problem solving dataset +described in the paper "Evaluating Large Language Models Trained on Code" +(https://arxiv.org/abs/2107.03374).""" + +import itertools +import os +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed + +import datasets +import numpy as np + +import evaluate + +from .execute import check_correctness + + +_CITATION = """\ +@misc{chen2021evaluating, + title={Evaluating Large Language Models Trained on Code}, + author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ +and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ +and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ +and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ +and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ +and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ +and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ +and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ +and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ +and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ +and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ +and William Saunders and Christopher Hesse and Andrew N. Carr \ +and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ +and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ +and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ +and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, + year={2021}, + eprint={2107.03374}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +""" + +_DESCRIPTION = """\ +This metric implements the evaluation harness for the HumanEval problem solving dataset +described in the paper "Evaluating Large Language Models Trained on Code" +(https://arxiv.org/abs/2107.03374). +""" + + +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores +Args: + predictions: list of candidates to evaluate. Each candidates should be a list + of strings with several code candidates to solve the problem. + references: a list with a test for each prediction. Each test should evaluate the + correctness of a code candidate. + k: number of code candidates to consider in the evaluation (Default: [1, 10, 100]) + num_workers: number of workers used to evaluate the canidate programs (Default: 4). + timeout: +Returns: + pass_at_k: dict with pass rates for each k + results: dict with granular results of each unittest +Examples: + >>> code_eval = evaluate.load("code_eval") + >>> test_cases = ["assert add(2,3)==5"] + >>> candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] + >>> pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) + >>> print(pass_at_k) + {'pass@1': 0.5, 'pass@2': 1.0} +""" + + +_WARNING = """ +################################################################################ + !!!WARNING!!! +################################################################################ +The "code_eval" metric executes untrusted model-generated code in Python. +Although it is highly unlikely that model-generated code will do something +overtly malicious in response to this test suite, model-generated code may act +destructively due to a lack of model capability or alignment. +Users are strongly encouraged to sandbox this evaluation suite so that it +does not perform destructive actions on their host or network. For more +information on how OpenAI sandboxes its code, see the paper "Evaluating Large +Language Models Trained on Code" (https://arxiv.org/abs/2107.03374). + +Once you have read this disclaimer and taken appropriate precautions, +set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this +with: + +>>> import os +>>> os.environ["HF_ALLOW_CODE_EVAL"] = "1" + +################################################################################\ +""" + +_LICENSE = """The MIT License + +Copyright (c) OpenAI (https://openai.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CodeEval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + # This is the description that will appear on the metrics page. + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string")), + "references": datasets.Value("string"), + } + ), + homepage="https://github.com/openai/human-eval", + codebase_urls=["https://github.com/openai/human-eval"], + reference_urls=["https://github.com/openai/human-eval"], + license=_LICENSE, + ) + + def _compute(self, predictions, references, k=[1, 10, 100], num_workers=4, timeout=3.0): + """Returns the scores""" + + if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1": + raise ValueError(_WARNING) + + if os.name == "nt": + raise NotImplementedError("This metric is currently not supported on Windows.") + + with ThreadPoolExecutor(max_workers=num_workers) as executor: + futures = [] + completion_id = Counter() + n_samples = 0 + results = defaultdict(list) + + for task_id, (candidates, test_case) in enumerate(zip(predictions, references)): + for candidate in candidates: + test_program = candidate + "\n" + test_case + args = (test_program, timeout, task_id, completion_id[task_id]) + future = executor.submit(check_correctness, *args) + futures.append(future) + completion_id[task_id] += 1 + n_samples += 1 + + for future in as_completed(futures): + result = future.result() + results[result["task_id"]].append((result["completion_id"], result)) + + total, correct = [], [] + for result in results.values(): + result.sort() + passed = [r[1]["passed"] for r in result] + total.append(len(passed)) + correct.append(sum(passed)) + total = np.array(total) + correct = np.array(correct) + + ks = k + pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean() for k in ks if (total >= k).all()} + + return pass_at_k, results + + +def estimate_pass_at_k(num_samples, num_correct, k): + """Estimates pass@k of each problem and returns them in an array.""" + + def estimator(n: int, c: int, k: int) -> float: + """Calculates 1 - comb(n - c, k) / comb(n, k).""" + if n - c < k: + return 1.0 + return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) + + if isinstance(num_samples, int): + num_samples_it = itertools.repeat(num_samples, len(num_correct)) + else: + assert len(num_samples) == len(num_correct) + num_samples_it = iter(num_samples) + + return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) diff --git a/ST/evaluate/metrics/code_eval/execute.py b/ST/evaluate/metrics/code_eval/execute.py new file mode 100644 index 0000000000000000000000000000000000000000..53517a805cf84758b858612b62794b874590159b --- /dev/null +++ b/ST/evaluate/metrics/code_eval/execute.py @@ -0,0 +1,236 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code is adapted from OpenAI's release +# https://github.com/openai/human-eval/blob/master/human_eval/execution.py + +import contextlib +import faulthandler +import io +import multiprocessing +import os +import platform +import signal +import tempfile + + +def check_correctness(check_program, timeout, task_id, completion_id): + """ + Evaluates the functional correctness of a completion by running the test + suite provided in the problem. + + :param completion_id: an optional completion ID so we can match + the results later even if execution finishes asynchronously. + """ + manager = multiprocessing.Manager() + result = manager.list() + + p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout)) + p.start() + p.join(timeout=timeout + 1) + if p.is_alive(): + p.kill() + + if not result: + result.append("timed out") + + return dict( + task_id=task_id, + passed=result[0] == "passed", + result=result[0], + completion_id=completion_id, + ) + + +def unsafe_execute(check_program, result, timeout): + + with create_tempdir(): + + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + + # Disable functionalities that can make destructive changes to the test. + reliability_guard() + + # Run program. + try: + exec_globals = {} + with swallow_io(): + with time_limit(timeout): + exec(check_program, exec_globals) + result.append("passed") + except TimeoutException: + result.append("timed out") + except BaseException as e: + result.append(f"failed: {e}") + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + +@contextlib.contextmanager +def time_limit(seconds): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +@contextlib.contextmanager +def swallow_io(): + stream = WriteOnlyStringIO() + with contextlib.redirect_stdout(stream): + with contextlib.redirect_stderr(stream): + with redirect_stdin(stream): + yield + + +@contextlib.contextmanager +def create_tempdir(): + with tempfile.TemporaryDirectory() as dirname: + with chdir(dirname): + yield dirname + + +class TimeoutException(Exception): + pass + + +class WriteOnlyStringIO(io.StringIO): + """StringIO that throws an exception when it's read from""" + + def read(self, *args, **kwargs): + raise OSError + + def readline(self, *args, **kwargs): + raise OSError + + def readlines(self, *args, **kwargs): + raise OSError + + def readable(self, *args, **kwargs): + """Returns True if the IO object can be read.""" + return False + + +class redirect_stdin(contextlib._RedirectStream): # type: ignore + _stream = "stdin" + + +@contextlib.contextmanager +def chdir(root): + if root == ".": + yield + return + cwd = os.getcwd() + os.chdir(root) + try: + yield + except BaseException as exc: + raise exc + finally: + os.chdir(cwd) + + +def reliability_guard(maximum_memory_bytes=None): + """ + This disables various destructive functions and prevents the generated code + from interfering with the test (e.g. fork bomb, killing other processes, + removing filesystem files, etc.) + + WARNING + This function is NOT a security sandbox. Untrusted code, including, model- + generated code, should not be blindly executed outside of one. See the + Codex paper for more information about OpenAI's code sandbox, and proceed + with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == "Darwin": + resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + import builtins + + builtins.exit = None + builtins.quit = None + + import os + + os.environ["OMP_NUM_THREADS"] = "1" + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + __builtins__["help"] = None + + import sys + + sys.modules["ipdb"] = None + sys.modules["joblib"] = None + sys.modules["resource"] = None + sys.modules["psutil"] = None + sys.modules["tkinter"] = None diff --git a/ST/evaluate/metrics/code_eval/requirements.txt b/ST/evaluate/metrics/code_eval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/code_eval/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/comet/README.md b/ST/evaluate/metrics/comet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..999f3fc44eb3b3e90408346fa744865dd9e9cb7c --- /dev/null +++ b/ST/evaluate/metrics/comet/README.md @@ -0,0 +1,204 @@ +--- +title: COMET +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM). + With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition. + + See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information. +--- + +# Metric Card for COMET + +## Metric description + +Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments. + +## How to use + +COMET takes 3 lists of strings as input: `sources` (a list of source sentences), `predictions` (a list of candidate translations) and `references` (a list of reference translations). + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] +comet_score = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +``` + +It has several configurations, named after the COMET model to be used. For versions below 2.0 it will default to `wmt20-comet-da` (previously known as `wmt-large-da-estimator-1719`) and for the latest versions (>= 2.0) it will default to `Unbabel/wmt22-comet-da`. + +Alternative models that can be chosen include `wmt20-comet-qe-da`, `wmt21-comet-mqm`, `wmt21-cometinho-da`, `wmt21-comet-qe-mqm` and `emnlp20-comet-rank`. Notably, a distilled model is also available, which is 80% smaller and 2.128x faster while performing close to non-distilled alternatives. You can use it with the identifier `eamt22-cometinho-da`. This version, called Cometinho, was elected as [the best paper](https://aclanthology.org/2022.eamt-1.9) at the annual European conference on Machine Translation. + +> NOTE: In `unbabel-comet>=2.0` all models were moved to Hugging Face Hub and you need to add the suffix `Unbabel/` to be able to download and use them. For example for the distilled version replace `eamt22-cometinho-da` with `Unbabel/eamt22-cometinho-da`. + +It also has several optional arguments: + +`gpus`: optional, an integer (number of GPUs to train on) or a list of integers (which GPUs to train on). Set to 0 to use CPU. The default value is `None` (uses one GPU if possible, else use CPU). + +`progress_bar`a boolean -- if set to `True`, progress updates will be printed out. The default value is `False`. + +More information about model characteristics can be found on the [COMET website](https://unbabel.github.io/COMET/html/index.html). + +## Output values + +The COMET metric outputs two lists: + +`scores`: a list of COMET scores for each of the input sentences, ranging from 0-1. + +`mean_score`: the mean value of COMET scores `scores` over all the input sentences, ranging from 0-1. + +### Values from popular papers + +The [original COMET paper](https://arxiv.org/pdf/2009.09025.pdf) reported average COMET scores ranging from 0.4 to 0.6, depending on the language pairs used for evaluating translation models. They also illustrate that COMET correlates well with human judgements compared to other metrics such as [BLEU](https://huggingface.co/metrics/bleu) and [CHRF](https://huggingface.co/metrics/chrf). + +## Examples + +Full match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["They were able to control the fire.", "Schools and kindergartens opened"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 1) for v in results["scores"]]) +[1.0, 1.0] +``` + +Partial match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 2) for v in results["scores"]]) +[0.19, 0.92] +``` + +No match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The girl went for a walk", "The boy was sleeping"] +reference = ["They were able to control the fire", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 2) for v in results["scores"]]) +[0.00, 0.00] +``` + +## Limitations and bias + +The models provided for calculating the COMET metric are built on top of XLM-R and cover the following languages: + +Afrikaans, Albanian, Amharic, Arabic, Armenian, Assamese, Azerbaijani, Basque, Belarusian, Bengali, Bengali Romanized, Bosnian, Breton, Bulgarian, Burmese, Burmese, Catalan, Chinese (Simplified), Chinese (Traditional), Croatian, Czech, Danish, Dutch, English, Esperanto, Estonian, Filipino, Finnish, French, Galician, Georgian, German, Greek, Gujarati, Hausa, Hebrew, Hindi, Hindi Romanized, Hungarian, Icelandic, Indonesian, Irish, Italian, Japanese, Javanese, Kannada, Kazakh, Khmer, Korean, Kurdish (Kurmanji), Kyrgyz, Lao, Latin, Latvian, Lithuanian, Macedonian, Malagasy, Malay, Malayalam, Marathi, Mongolian, Nepali, Norwegian, Oriya, Oromo, Pashto, Persian, Polish, Portuguese, Punjabi, Romanian, Russian, Sanskri, Scottish, Gaelic, Serbian, Sindhi, Sinhala, Slovak, Slovenian, Somali, Spanish, Sundanese, Swahili, Swedish, Tamil, Tamil Romanized, Telugu, Telugu Romanized, Thai, Turkish, Ukrainian, Urdu, Urdu Romanized, Uyghur, Uzbek, Vietnamese, Welsh, Western, Frisian, Xhosa, Yiddish. + +Thus, results for language pairs containing uncovered languages are unreliable, as per the [COMET website](https://github.com/Unbabel/COMET) + +Also, calculating the COMET metric involves downloading the model from which features are obtained -- the default model, `wmt22-comet-da`, takes over 2.32GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `eamt22-cometinho-da` is 344MB. + +### Interpreting Scores: + +When using COMET to evaluate machine translation, it's important to understand how to interpret the scores it produces. + +In general, COMET models are trained to predict quality scores for translations. These scores are typically normalized using a z-score transformation to account for individual differences among annotators. While the raw score itself does not have a direct interpretation, it is useful for ranking translations and systems according to their quality. + +However, for the latest COMET models like `Unbabel/wmt22-comet-da`, we have introduced a new training approach that scales the scores between 0 and 1. This makes it easier to interpret the scores: a score close to 1 indicates a high-quality translation, while a score close to 0 indicates a translation that is no better than random chance. + +It's worth noting that when using COMET to compare the performance of two different translation systems, it's important to run statistical significance measures to reliably compare scores between systems. + +## Citation +```bibtex +@inproceedings{rei-etal-2022-comet, + title = "{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task", + author = "Rei, Ricardo and + C. de Souza, Jos{\'e} G. and + Alves, Duarte and + Zerva, Chrysoula and + Farinha, Ana C and + Glushkova, Taisiya and + Lavie, Alon and + Coheur, Luisa and + Martins, Andr{\'e} F. T.", + booktitle = "Proceedings of the Seventh Conference on Machine Translation (WMT)", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.wmt-1.52", + pages = "578--585", +} +``` + +```bibtex +@inproceedings{rei-EtAl:2020:WMT, + author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, + title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, + booktitle = {Proceedings of the Fifth Conference on Machine Translation}, + month = {November}, + year = {2020}, + address = {Online}, + publisher = {Association for Computational Linguistics}, + pages = {909--918}, +} +``` + +```bibtex +@inproceedings{rei-etal-2020-comet, + title = "{COMET}: A Neural Framework for {MT} Evaluation", + author = "Rei, Ricardo and + Stewart, Craig and + Farinha, Ana C and + Lavie, Alon", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", + pages = "2685--2702", +} +``` + +For the distilled version: + +```bibtex +@inproceedings{rei-etal-2022-searching, + title = "Searching for {COMETINHO}: The Little Metric That Could", + author = "Rei, Ricardo and + Farinha, Ana C and + de Souza, Jos{\'e} G.C. and + Ramos, Pedro G. and + Martins, Andr{\'e} F.T. and + Coheur, Luisa and + Lavie, Alon", + booktitle = "Proceedings of the 23rd Annual Conference of the European Association for Machine Translation", + month = jun, + year = "2022", + address = "Ghent, Belgium", + publisher = "European Association for Machine Translation", + url = "https://aclanthology.org/2022.eamt-1.9", + pages = "61--70", +} +``` + +## Further References + +- [COMET website](https://unbabel.github.io/COMET/html/index.html) +- [Hugging Face Tasks - Machine Translation](https://huggingface.co/tasks/translation) diff --git a/ST/evaluate/metrics/comet/app.py b/ST/evaluate/metrics/comet/app.py new file mode 100644 index 0000000000000000000000000000000000000000..629cf5375bdff6520dcd7c86dbad8416ea1150b3 --- /dev/null +++ b/ST/evaluate/metrics/comet/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("comet") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/comet/comet.py b/ST/evaluate/metrics/comet/comet.py new file mode 100644 index 0000000000000000000000000000000000000000..33953b07314852735bbe6c04760fe886fb3b6fae --- /dev/null +++ b/ST/evaluate/metrics/comet/comet.py @@ -0,0 +1,171 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" COMET metric. + +Requirements: +pip install unbabel-comet + +Usage: + +```python +from evaluate import load +comet_metric = load('metrics/comet/comet.py') +#comet_metric = load('comet') +#comet_metric = load('comet', 'Unbabel/wmt20-comet-da') + + +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] + +predictions = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +predictions['scores'] +``` +""" + +import comet # From: unbabel-comet +import datasets +import torch +from packaging import version + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +_CITATION = """\ +@inproceedings{rei-etal-2022-comet, + title = "{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task", + author = "Rei, Ricardo and + C. de Souza, Jos{\'e} G. and + Alves, Duarte and + Zerva, Chrysoula and + Farinha, Ana C and + Glushkova, Taisiya and + Lavie, Alon and + Coheur, Luisa and + Martins, Andr{\'e} F. T.", + booktitle = "Proceedings of the Seventh Conference on Machine Translation (WMT)", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.wmt-1.52", + pages = "578--585", +} +@inproceedings{rei-EtAl:2020:WMT, + author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, + title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, + booktitle = {Proceedings of the Fifth Conference on Machine Translation}, + month = {November}, + year = {2020}, + address = {Online}, + publisher = {Association for Computational Linguistics}, + pages = {909--918}, +} +@inproceedings{rei-etal-2020-comet, + title = "{COMET}: A Neural Framework for {MT} Evaluation", + author = "Rei, Ricardo and + Stewart, Craig and + Farinha, Ana C and + Lavie, Alon", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", + pages = "2685--2702", +} +""" + +_DESCRIPTION = """\ +Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM). +With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition. + +See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information. +""" + +_KWARGS_DESCRIPTION = """ +COMET score. + +Args: + +`sources` (list of str): Source sentences +`predictions` (list of str): candidate translations +`references` (list of str): reference translations +`gpus` (bool): Number of GPUs to use. 0 for CPU +`progress_bar` (bool): Flag that turns on and off the predict progress bar. Defaults to True + +Returns: + Dict with all sentence-level scores (`scores` key) a system-level score (`mean_score` key). + +Examples: + + >>> comet_metric = evaluate.load('comet') + >>> # comet_metric = load('comet', 'wmt20-comet-da') # you can also choose which model to use + >>> source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] + >>> hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] + >>> reference = ["They were able to control the fire.", "Schools and kindergartens opened"] + >>> results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) + >>> print([round(v, 3) for v in results["scores"]]) + [0.839, 0.972] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class COMET(evaluate.Metric): + def _info(self): + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://unbabel.github.io/COMET/html/index.html", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "sources": datasets.Value("string", id="sequence"), + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/Unbabel/COMET"], + reference_urls=[ + "https://github.com/Unbabel/COMET", + "https://aclanthology.org/2022.wmt-1.52/", + "https://www.aclweb.org/anthology/2020.emnlp-main.213/", + "http://www.statmt.org/wmt20/pdf/2020.wmt-1.101.pdf6", + ], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + if version.parse(comet.__version__) >= version.parse("2.0.0"): + self.scorer = comet.load_from_checkpoint(comet.download_model("Unbabel/wmt22-comet-da")) + else: + self.scorer = comet.load_from_checkpoint(comet.download_model("wmt20-comet-da")) + else: + self.scorer = comet.load_from_checkpoint(comet.download_model(self.config_name)) + + def _compute(self, sources, predictions, references, gpus=None, progress_bar=False): + if gpus is None: + gpus = 1 if torch.cuda.is_available() else 0 + data = {"src": sources, "mt": predictions, "ref": references} + data = [dict(zip(data, t)) for t in zip(*data.values())] + if version.parse(comet.__version__) >= version.parse("2.0.0"): + output = self.scorer.predict(data, gpus=gpus, progress_bar=progress_bar) + scores, mean_score = output.scores, output.system_score + else: + scores, mean_score = self.scorer.predict(data, gpus=gpus, progress_bar=progress_bar) + return {"mean_score": mean_score, "scores": scores} diff --git a/ST/evaluate/metrics/comet/requirements.txt b/ST/evaluate/metrics/comet/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c50454c31606eb0b885ef4669bb785173d467dd9 --- /dev/null +++ b/ST/evaluate/metrics/comet/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +unbabel-comet +torch \ No newline at end of file diff --git a/ST/evaluate/metrics/competition_math/README.md b/ST/evaluate/metrics/competition_math/README.md new file mode 100644 index 0000000000000000000000000000000000000000..44a53000472750fbac5006656345038768a0db12 --- /dev/null +++ b/ST/evaluate/metrics/competition_math/README.md @@ -0,0 +1,120 @@ +--- +title: Competition MATH +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset. + It first canonicalizes the inputs (e.g., converting "1/2" to "\frac{1}{2}") and then computes accuracy. +--- + +# Metric Card for Competition MATH + +## Metric description + +This metric is used to assess performance on the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math). + +It first canonicalizes the inputs (e.g., converting `1/2` to `\\frac{1}{2}`) and then computes accuracy. + +## How to use + +This metric takes two arguments: + +`predictions`: a list of predictions to score. Each prediction is a string that contains natural language and LaTeX. + +`references`: list of reference for each prediction. Each reference is a string that contains natural language and LaTeX. + + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["1/2"] +>>> results = math.compute(references=references, predictions=predictions) +``` + +N.B. To be able to use Competition MATH, you need to install the `math_equivalence` dependency using `pip install git+https://github.com/hendrycks/math.git`. + + +## Output values + +This metric returns a dictionary that contains the [accuracy](https://huggingface.co/metrics/accuracy) after canonicalizing inputs, on a scale between 0.0 and 1.0. + +### Values from popular papers +The [original MATH dataset paper](https://arxiv.org/abs/2103.03874) reported accuracies ranging from 3.0% to 6.9% by different large language models. + +More recent progress on the dataset can be found on the [dataset leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math). + +## Examples + +Maximal values (full match): + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["1/2"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 1.0} +``` + +Minimal values (no match): + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["3/4"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 0.0} +``` + +Partial match: + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}","\\frac{3}{4}"] +>>> predictions = ["1/5", "3/4"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 0.5} +``` + +## Limitations and bias + +This metric is limited to datasets with the same format as the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math), and is meant to evaluate the performance of large language models at solving mathematical problems. + +N.B. The MATH dataset also assigns levels of difficulty to different problems, so disagregating model performance by difficulty level (similarly to what was done in the [original paper](https://arxiv.org/abs/2103.03874) can give a better indication of how a given model does on a given difficulty of math problem, compared to overall accuracy. + +## Citation + +```bibtex +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks + and Collin Burns + and Saurav Kadavath + and Akul Arora + and Steven Basart + and Eric Tang + and Dawn Song + and Jacob Steinhardt}, + journal={arXiv preprint arXiv:2103.03874}, + year={2021} +} +``` + +## Further References +- [MATH dataset](https://huggingface.co/datasets/competition_math) +- [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math) +- [MATH paper](https://arxiv.org/abs/2103.03874) \ No newline at end of file diff --git a/ST/evaluate/metrics/competition_math/app.py b/ST/evaluate/metrics/competition_math/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0cbc7ed2c0ffe712cde35cef4fe86b1f9e4939c8 --- /dev/null +++ b/ST/evaluate/metrics/competition_math/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("competition_math") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/competition_math/competition_math.py b/ST/evaluate/metrics/competition_math/competition_math.py new file mode 100644 index 0000000000000000000000000000000000000000..9a82eb40b656dfe892aa5f3ebfef30c21329e92d --- /dev/null +++ b/ST/evaluate/metrics/competition_math/competition_math.py @@ -0,0 +1,95 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric for the Mathematics Aptitude Test of Heuristics (MATH) dataset.""" + +import datasets +import math_equivalence # From: git+https://github.com/hendrycks/math.git + +import evaluate + + +_CITATION = """\ +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks + and Collin Burns + and Saurav Kadavath + and Akul Arora + and Steven Basart + and Eric Tang + and Dawn Song + and Jacob Steinhardt}, + journal={arXiv preprint arXiv:2103.03874}, + year={2021} +} +""" + + +_DESCRIPTION = """\ +This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset. +It first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy. +""" + + +_KWARGS_DESCRIPTION = r""" +Calculates accuracy after canonicalizing inputs. + +Args: + predictions: list of predictions to score. Each prediction + is a string that contains natural language and LaTex. + references: list of reference for each prediction. Each + reference is a string that contains natural language + and LaTex. +Returns: + accuracy: accuracy after canonicalizing inputs + (e.g., converting "1/2" to "\\frac{1}{2}") + +Examples: + >>> metric = evaluate.load("competition_math") + >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"]) + >>> print(results) + {'accuracy': 1.0} +""" + + +@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CompetitionMathMetric(evaluate.Metric): + """Accuracy metric for the MATH dataset.""" + + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + "references": datasets.Value("string"), + } + ), + # Homepage of the metric for documentation + homepage="https://github.com/hendrycks/math", + # Additional links to the codebase or references + codebase_urls=["https://github.com/hendrycks/math"], + ) + + def _compute(self, predictions, references): + """Returns the scores""" + n_correct = 0.0 + for i, j in zip(predictions, references): + n_correct += 1.0 if math_equivalence.is_equiv(i, j) else 0.0 + accuracy = n_correct / len(predictions) + return { + "accuracy": accuracy, + } diff --git a/ST/evaluate/metrics/competition_math/requirements.txt b/ST/evaluate/metrics/competition_math/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1d8f9f406828016d38bbc88efd96848edd1b0013 --- /dev/null +++ b/ST/evaluate/metrics/competition_math/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/hendrycks/math.git \ No newline at end of file diff --git a/ST/evaluate/metrics/coval/README.md b/ST/evaluate/metrics/coval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7405f729dcab78fef1582199a2821cd367fe156b --- /dev/null +++ b/ST/evaluate/metrics/coval/README.md @@ -0,0 +1,215 @@ +--- +title: +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + CoVal is a coreference evaluation tool for the CoNLL and ARRAU datasets which + implements of the common evaluation metrics including MUC [Vilain et al, 1995], + B-cubed [Bagga and Baldwin, 1998], CEAFe [Luo et al., 2005], + LEA [Moosavi and Strube, 2016] and the averaged CoNLL score + (the average of the F1 values of MUC, B-cubed and CEAFe) + [Denis and Baldridge, 2009a; Pradhan et al., 2011]. + + This wrapper of CoVal currently only work with CoNLL line format: + The CoNLL format has one word per line with all the annotation for this word in column separated by spaces: + Column Type Description + 1 Document ID This is a variation on the document filename + 2 Part number Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. + 3 Word number + 4 Word itself This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. + 5 Part-of-Speech + 6 Parse bit This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. + 7 Predicate lemma The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-" + 8 Predicate Frameset ID This is the PropBank frameset ID of the predicate in Column 7. + 9 Word sense This is the word sense of the word in Column 3. + 10 Speaker/Author This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. + 11 Named Entities These columns identifies the spans representing various named entities. + 12:N Predicate Arguments There is one column each of predicate argument structure information for the predicate mentioned in Column 7. + N Coreference Coreference chain information encoded in a parenthesis structure. + More informations on the format can be found here (section "*_conll File Format"): http://www.conll.cemantix.org/2012/data.html + + Details on the evaluation on CoNLL can be found here: https://github.com/ns-moosavi/coval/blob/master/conll/README.md + + CoVal code was written by @ns-moosavi. + Some parts are borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py + The test suite is taken from https://github.com/conll/reference-coreference-scorers/ + Mention evaluation and the test suite are added by @andreasvc. + Parsing CoNLL files is developed by Leo Born. +--- + +## Metric description + +CoVal is a coreference evaluation tool for the [CoNLL](https://huggingface.co/datasets/conll2003) and [ARRAU](https://catalog.ldc.upenn.edu/LDC2013T22) datasets which implements of the common evaluation metrics including MUC [Vilain et al, 1995](https://aclanthology.org/M95-1005.pdf), B-cubed [Bagga and Baldwin, 1998](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.34.2578&rep=rep1&type=pdf), CEAFe [Luo et al., 2005](https://aclanthology.org/H05-1004.pdf), LEA [Moosavi and Strube, 2016](https://aclanthology.org/P16-1060.pdf) and the averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe). + +CoVal code was written by [`@ns-moosavi`](https://github.com/ns-moosavi), with some parts borrowed from [Deep Coref](https://github.com/clarkkev/deep-coref/blob/master/evaluation.py). The test suite is taken from the [official CoNLL code](https://github.com/conll/reference-coreference-scorers/), with additions by [`@andreasvc`](https://github.com/andreasvc) and file parsing developed by Leo Born. + +## How to use + +The metric takes two lists of sentences as input: one representing `predictions` and `references`, with the sentences consisting of words in the CoNLL format (see the [Limitations and bias](#Limitations-and-bias) section below for more details on the CoNLL format). + +```python +from evaluate import load +coval = load('coval') +words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', +... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', +... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', +... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', +... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', +... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] +references = [words] +predictions = [words] +results = coval.compute(predictions=predictions, references=references) +``` +It also has several optional arguments: + +`keep_singletons`: After extracting all mentions of key or system file mentions whose corresponding coreference chain is of size one are considered as singletons. The default evaluation mode will include singletons in evaluations if they are included in the key or the system files. By setting `keep_singletons=False`, all singletons in the key and system files will be excluded from the evaluation. + +`NP_only`: Most of the recent coreference resolvers only resolve NP mentions and leave out the resolution of VPs. By setting the `NP_only` option, the scorer will only evaluate the resolution of NPs. + +`min_span`: By setting `min_span`, the scorer reports the results based on automatically detected minimum spans. Minimum spans are determined using the [MINA algorithm](https://arxiv.org/pdf/1906.06703.pdf). + + +## Output values + +The metric outputs a dictionary with the following key-value pairs: + +`mentions`: number of mentions, ranges from 0-1 + +`muc`: MUC metric, which expresses performance in terms of recall and precision, ranging from 0-1. + +`bcub`: B-cubed metric, which is the averaged precision of all items in the distribution, ranging from 0-1. + +`ceafe`: CEAFe (Constrained Entity Alignment F-Measure) is computed by aligning reference and system entities with the constraint that a reference entity is aligned with at most one reference entity. It ranges from 0-1 + +`lea`: LEA is a Link-Based Entity-Aware metric which, for each entity, considers how important the entity is and how well it is resolved. It ranges from 0-1. + +`conll_score`: averaged CoNLL score (the average of the F1 values of `muc`, `bcub` and `ceafe`), ranging from 0 to 100. + + +### Values from popular papers + +Given that many of the metrics returned by COVAL come from different sources, is it hard to cite reference values for all of them. + +The CoNLL score is used to track progress on different datasets such as the [ARRAU corpus](https://paperswithcode.com/sota/coreference-resolution-on-the-arrau-corpus) and [CoNLL 2012](https://paperswithcode.com/sota/coreference-resolution-on-conll-2012). + +## Examples + +Maximal values + +```python +from evaluate import load +coval = load('coval') +words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', +... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', +... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', +... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', +... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', +... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] +references = [words] +predictions = [words] +results = coval.compute(predictions=predictions, references=references) +print(results) +{'mentions/recall': 1.0, 'mentions/precision': 1.0, 'mentions/f1': 1.0, 'muc/recall': 1.0, 'muc/precision': 1.0, 'muc/f1': 1.0, 'bcub/recall': 1.0, 'bcub/precision': 1.0, 'bcub/f1': 1.0, 'ceafe/recall': 1.0, 'ceafe/precision': 1.0, 'ceafe/f1': 1.0, 'lea/recall': 1.0, 'lea/precision': 1.0, 'lea/f1': 1.0, 'conll_score': 100.0} +``` + +## Limitations and bias + +This wrapper of CoVal currently only works with [CoNLL line format](https://huggingface.co/datasets/conll2003), which has one word per line with all the annotation for this word in column separated by spaces: + +| Column | Type | Description | +|:-------|:----------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | Document ID | This is a variation on the document filename | +| 2 | Part number | Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. | +| 3 | Word number | | +| 4 | Word | This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. | +| 5 | Part-of-Speech | | +| 6 | Parse bit | This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. | +| 7 | Predicate lemma | The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-". | +| 8 | Predicate Frameset ID | This is the PropBank frameset ID of the predicate in Column 7. | +| 9 | Word sense | This is the word sense of the word in Column 3. | +| 10 | Speaker/Author | This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. | +| 11 | Named Entities | These columns identifies the spans representing various named entities. | +| 12:N | Predicate Arguments | There is one column each of predicate argument structure information for the predicate mentioned in Column 7. | +| N | Coreference | Coreference chain information encoded in a parenthesis structure. | + +## Citations + +```bibtex +@InProceedings{moosavi2019minimum, + author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, + title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, + year = {2019}, + booktitle = {Proceedings of the 57th Annual Meeting of + the Association for Computational Linguistics (Volume 1: Long Papers)}, + publisher = {Association for Computational Linguistics}, + address = {Florence, Italy}, +} +``` +```bibtex +@inproceedings{10.3115/1072399.1072405, + author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, + title = {A Model-Theoretic Coreference Scoring Scheme}, + year = {1995}, + isbn = {1558604022}, + publisher = {Association for Computational Linguistics}, + address = {USA}, + url = {https://doi.org/10.3115/1072399.1072405}, + doi = {10.3115/1072399.1072405}, + booktitle = {Proceedings of the 6th Conference on Message Understanding}, + pages = {45–52}, + numpages = {8}, + location = {Columbia, Maryland}, + series = {MUC6 ’95} +} +``` + +```bibtex +@INPROCEEDINGS{Bagga98algorithmsfor, + author = {Amit Bagga and Breck Baldwin}, + title = {Algorithms for Scoring Coreference Chains}, + booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, + year = {1998}, + pages = {563--566} +} +``` +```bibtex +@INPROCEEDINGS{Luo05oncoreference, + author = {Xiaoqiang Luo}, + title = {On coreference resolution performance metrics}, + booktitle = {In Proc. of HLT/EMNLP}, + year = {2005}, + pages = {25--32}, + publisher = {URL} +} +``` + +```bibtex +@inproceedings{moosavi-strube-2016-coreference, + title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", + author = "Moosavi, Nafise Sadat and + Strube, Michael", + booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P16-1060", + doi = "10.18653/v1/P16-1060", + pages = "632--642", +} +``` + +## Further References + +- [CoNLL 2012 Task Description](http://www.conll.cemantix.org/2012/data.html): for information on the format (section "*_conll File Format") +- [CoNLL Evaluation details](https://github.com/ns-moosavi/coval/blob/master/conll/README.md) +- [Hugging Face - Neural Coreference Resolution (Neuralcoref)](https://huggingface.co/coref/) + diff --git a/ST/evaluate/metrics/coval/app.py b/ST/evaluate/metrics/coval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a2428d98685c3022ef7f9f57b7214d87d1f2984d --- /dev/null +++ b/ST/evaluate/metrics/coval/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("coval") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/coval/coval.py b/ST/evaluate/metrics/coval/coval.py new file mode 100644 index 0000000000000000000000000000000000000000..f1518b95852ee164b3851633f2716de80abb32d6 --- /dev/null +++ b/ST/evaluate/metrics/coval/coval.py @@ -0,0 +1,321 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CoVal metric. """ +import coval # From: git+https://github.com/ns-moosavi/coval.git noqa: F401 +import datasets +from coval.conll import reader, util +from coval.eval import evaluator + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """\ +@InProceedings{moosavi2019minimum, + author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, + title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, + year = {2019}, + booktitle = {Proceedings of the 57th Annual Meeting of + the Association for Computational Linguistics (Volume 1: Long Papers)}, + publisher = {Association for Computational Linguistics}, + address = {Florence, Italy}, +} + +@inproceedings{10.3115/1072399.1072405, +author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, +title = {A Model-Theoretic Coreference Scoring Scheme}, +year = {1995}, +isbn = {1558604022}, +publisher = {Association for Computational Linguistics}, +address = {USA}, +url = {https://doi.org/10.3115/1072399.1072405}, +doi = {10.3115/1072399.1072405}, +booktitle = {Proceedings of the 6th Conference on Message Understanding}, +pages = {45–52}, +numpages = {8}, +location = {Columbia, Maryland}, +series = {MUC6 ’95} +} + +@INPROCEEDINGS{Bagga98algorithmsfor, + author = {Amit Bagga and Breck Baldwin}, + title = {Algorithms for Scoring Coreference Chains}, + booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, + year = {1998}, + pages = {563--566} +} + +@INPROCEEDINGS{Luo05oncoreference, + author = {Xiaoqiang Luo}, + title = {On coreference resolution performance metrics}, + booktitle = {In Proc. of HLT/EMNLP}, + year = {2005}, + pages = {25--32}, + publisher = {URL} +} + +@inproceedings{moosavi-strube-2016-coreference, + title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", + author = "Moosavi, Nafise Sadat and + Strube, Michael", + booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P16-1060", + doi = "10.18653/v1/P16-1060", + pages = "632--642", +} + +""" + +_DESCRIPTION = """\ +CoVal is a coreference evaluation tool for the CoNLL and ARRAU datasets which +implements of the common evaluation metrics including MUC [Vilain et al, 1995], +B-cubed [Bagga and Baldwin, 1998], CEAFe [Luo et al., 2005], +LEA [Moosavi and Strube, 2016] and the averaged CoNLL score +(the average of the F1 values of MUC, B-cubed and CEAFe) +[Denis and Baldridge, 2009a; Pradhan et al., 2011]. + +This wrapper of CoVal currently only work with CoNLL line format: +The CoNLL format has one word per line with all the annotation for this word in column separated by spaces: +Column Type Description +1 Document ID This is a variation on the document filename +2 Part number Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. +3 Word number +4 Word itself This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. +5 Part-of-Speech +6 Parse bit This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. +7 Predicate lemma The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-" +8 Predicate Frameset ID This is the PropBank frameset ID of the predicate in Column 7. +9 Word sense This is the word sense of the word in Column 3. +10 Speaker/Author This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. +11 Named Entities These columns identifies the spans representing various named entities. +12:N Predicate Arguments There is one column each of predicate argument structure information for the predicate mentioned in Column 7. +N Coreference Coreference chain information encoded in a parenthesis structure. +More informations on the format can be found here (section "*_conll File Format"): http://www.conll.cemantix.org/2012/data.html + +Details on the evaluation on CoNLL can be found here: https://github.com/ns-moosavi/coval/blob/master/conll/README.md + +CoVal code was written by @ns-moosavi. +Some parts are borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py +The test suite is taken from https://github.com/conll/reference-coreference-scorers/ +Mention evaluation and the test suite are added by @andreasvc. +Parsing CoNLL files is developed by Leo Born. +""" + +_KWARGS_DESCRIPTION = """ +Calculates coreference evaluation metrics. +Args: + predictions: list of sentences. Each sentence is a list of word predictions to score in the CoNLL format. + Each prediction is a word with its annotations as a string made of columns joined with spaces. + Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) + See the details on the format in the description of the metric. + references: list of sentences. Each sentence is a list of word reference to score in the CoNLL format. + Each reference is a word with its annotations as a string made of columns joined with spaces. + Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) + See the details on the format in the description of the metric. + keep_singletons: After extracting all mentions of key or system files, + mentions whose corresponding coreference chain is of size one, + are considered as singletons. The default evaluation mode will include + singletons in evaluations if they are included in the key or the system files. + By setting 'keep_singletons=False', all singletons in the key and system files + will be excluded from the evaluation. + NP_only: Most of the recent coreference resolvers only resolve NP mentions and + leave out the resolution of VPs. By setting the 'NP_only' option, the scorer will only evaluate the resolution of NPs. + min_span: By setting 'min_span', the scorer reports the results based on automatically detected minimum spans. + Minimum spans are determined using the MINA algorithm. + +Returns: + 'mentions': mentions + 'muc': MUC metric [Vilain et al, 1995] + 'bcub': B-cubed [Bagga and Baldwin, 1998] + 'ceafe': CEAFe [Luo et al., 2005] + 'lea': LEA [Moosavi and Strube, 2016] + 'conll_score': averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe) + +Examples: + + >>> coval = evaluate.load('coval') + >>> words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', + ... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', + ... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', + ... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', + ... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', + ... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] + >>> references = [words] + >>> predictions = [words] + >>> results = coval.compute(predictions=predictions, references=references) + >>> print(results) # doctest:+ELLIPSIS + {'mentions/recall': 1.0,[...] 'conll_score': 100.0} +""" + + +def get_coref_infos( + key_lines, sys_lines, NP_only=False, remove_nested=False, keep_singletons=True, min_span=False, doc="dummy_doc" +): + + key_doc_lines = {doc: key_lines} + sys_doc_lines = {doc: sys_lines} + + doc_coref_infos = {} + + key_nested_coref_num = 0 + sys_nested_coref_num = 0 + key_removed_nested_clusters = 0 + sys_removed_nested_clusters = 0 + key_singletons_num = 0 + sys_singletons_num = 0 + + key_clusters, singletons_num = reader.get_doc_mentions(doc, key_doc_lines[doc], keep_singletons) + key_singletons_num += singletons_num + + if NP_only or min_span: + key_clusters = reader.set_annotated_parse_trees(key_clusters, key_doc_lines[doc], NP_only, min_span) + + sys_clusters, singletons_num = reader.get_doc_mentions(doc, sys_doc_lines[doc], keep_singletons) + sys_singletons_num += singletons_num + + if NP_only or min_span: + sys_clusters = reader.set_annotated_parse_trees(sys_clusters, key_doc_lines[doc], NP_only, min_span) + + if remove_nested: + nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(key_clusters, keep_singletons) + key_nested_coref_num += nested_mentions + key_removed_nested_clusters += removed_clusters + + nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(sys_clusters, keep_singletons) + sys_nested_coref_num += nested_mentions + sys_removed_nested_clusters += removed_clusters + + sys_mention_key_cluster = reader.get_mention_assignments(sys_clusters, key_clusters) + key_mention_sys_cluster = reader.get_mention_assignments(key_clusters, sys_clusters) + + doc_coref_infos[doc] = (key_clusters, sys_clusters, key_mention_sys_cluster, sys_mention_key_cluster) + + if remove_nested: + logger.info( + "Number of removed nested coreferring mentions in the key " + f"annotation: {key_nested_coref_num}; and system annotation: {sys_nested_coref_num}" + ) + logger.info( + "Number of resulting singleton clusters in the key " + f"annotation: {key_removed_nested_clusters}; and system annotation: {sys_removed_nested_clusters}" + ) + + if not keep_singletons: + logger.info( + f"{key_singletons_num:d} and {sys_singletons_num:d} singletons are removed from the key and system " + "files, respectively" + ) + + return doc_coref_infos + + +def compute_score(key_lines, sys_lines, metrics, NP_only, remove_nested, keep_singletons, min_span): + doc_coref_infos = get_coref_infos(key_lines, sys_lines, NP_only, remove_nested, keep_singletons, min_span) + + output_scores = {} + conll = 0 + conll_subparts_num = 0 + + for name, metric in metrics: + recall, precision, f1 = evaluator.evaluate_documents(doc_coref_infos, metric, beta=1) + if name in ["muc", "bcub", "ceafe"]: + conll += f1 + conll_subparts_num += 1 + output_scores.update({f"{name}/recall": recall, f"{name}/precision": precision, f"{name}/f1": f1}) + + logger.info( + name.ljust(10), + f"Recall: {recall * 100:.2f}", + f" Precision: {precision * 100:.2f}", + f" F1: {f1 * 100:.2f}", + ) + + if conll_subparts_num == 3: + conll = (conll / 3) * 100 + logger.info(f"CoNLL score: {conll:.2f}") + output_scores.update({"conll_score": conll}) + + return output_scores + + +def check_gold_parse_annotation(key_lines): + has_gold_parse = False + for line in key_lines: + if not line.startswith("#"): + if len(line.split()) > 6: + parse_col = line.split()[5] + if not parse_col == "-": + has_gold_parse = True + break + else: + break + return has_gold_parse + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Coval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string")), + "references": datasets.Sequence(datasets.Value("string")), + } + ), + codebase_urls=["https://github.com/ns-moosavi/coval"], + reference_urls=[ + "https://github.com/ns-moosavi/coval", + "https://www.aclweb.org/anthology/P16-1060", + "http://www.conll.cemantix.org/2012/data.html", + ], + ) + + def _compute( + self, predictions, references, keep_singletons=True, NP_only=False, min_span=False, remove_nested=False + ): + allmetrics = [ + ("mentions", evaluator.mentions), + ("muc", evaluator.muc), + ("bcub", evaluator.b_cubed), + ("ceafe", evaluator.ceafe), + ("lea", evaluator.lea), + ] + + if min_span: + has_gold_parse = util.check_gold_parse_annotation(references) + if not has_gold_parse: + raise NotImplementedError("References should have gold parse annotation to use 'min_span'.") + # util.parse_key_file(key_file) + # key_file = key_file + ".parsed" + + score = compute_score( + key_lines=references, + sys_lines=predictions, + metrics=allmetrics, + NP_only=NP_only, + remove_nested=remove_nested, + keep_singletons=keep_singletons, + min_span=min_span, + ) + + return score diff --git a/ST/evaluate/metrics/coval/requirements.txt b/ST/evaluate/metrics/coval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c54425870ee74020ea4eadc825406dd62b09bb4 --- /dev/null +++ b/ST/evaluate/metrics/coval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/ns-moosavi/coval.git \ No newline at end of file diff --git a/ST/evaluate/metrics/cuad/README.md b/ST/evaluate/metrics/cuad/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9ae25a2b1d8ad9b3914f7748b59fb095ac29e7f4 --- /dev/null +++ b/ST/evaluate/metrics/cuad/README.md @@ -0,0 +1,132 @@ +--- +title: CUAD +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 1 of the Contract Understanding Atticus Dataset (CUAD). + + Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510 + commercial legal contracts that have been manually labeled to identify 41 categories of important + clauses that lawyers look for when reviewing contracts in connection with corporate transactions. +--- + +# Metric Card for CUAD + +## Metric description + +This metric wraps the official scoring script for version 1 of the [Contract Understanding Atticus Dataset (CUAD)](https://huggingface.co/datasets/cuad), which is a corpus of more than 13,000 labels in 510 commercial legal contracts that have been manually labeled to identify 41 categories of important clauses that lawyers look for when reviewing contracts in connection with corporate transactions. + +The CUAD metric computes several scores: [Exact Match](https://huggingface.co/metrics/exact_match), [F1 score](https://huggingface.co/metrics/f1), Area Under the Precision-Recall Curve, [Precision](https://huggingface.co/metrics/precision) at 80% [recall](https://huggingface.co/metrics/recall) and Precision at 90% recall. + +## How to use + +The CUAD metric takes two inputs : + + +`predictions`, a list of question-answer dictionaries with the following key-values: +- `id`: the id of the question-answer pair as given in the references. +- `prediction_text`: a list of possible texts for the answer, as a list of strings depending on a threshold on the confidence probability of each prediction. + + +`references`: a list of question-answer dictionaries with the following key-values: + - `id`: the id of the question-answer pair (the same as above). + - `answers`: a dictionary *in the CUAD dataset format* with the following keys: + - `text`: a list of possible texts for the answer, as a list of strings. + - `answer_start`: a list of start positions for the answer, as a list of ints. + + Note that `answer_start` values are not taken into account to compute the metric. + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the CUAD metric consists of a dictionary that contains one or several of the following metrics: + +`exact_match`: The normalized answers that exactly match the reference answer, with a range between 0.0 and 1.0 (see [exact match](https://huggingface.co/metrics/exact_match) for more information). + +`f1`: The harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is between 0.0 and 1.0 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`aupr`: The Area Under the Precision-Recall curve, with a range between 0.0 and 1.0, with a higher value representing both high recall and high precision, and a low value representing low values for both. See the [Wikipedia article](https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve) for more information. + +`prec_at_80_recall`: The fraction of true examples among the predicted examples at a recall rate of 80%. Its range is between 0.0 and 1.0. For more information, see [precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall). + +`prec_at_90_recall`: The fraction of true examples among the predicted examples at a recall rate of 90%. Its range is between 0.0 and 1.0. + + +### Values from popular papers +The [original CUAD paper](https://arxiv.org/pdf/2103.06268.pdf) reports that a [DeBERTa model](https://huggingface.co/microsoft/deberta-base) attains +an AUPR of 47.8%, a Precision at 80% Recall of 44.0%, and a Precision at 90% Recall of 17.8% (they do not report F1 or Exact Match separately). + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/cuad). + +## Examples + +Maximal values : + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} +``` + +Minimal values: + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.'], 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] +references = [{'answers': {'answer_start': [143], 'text': 'The seller'}, 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 0.0, 'f1': 0.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} +``` + +Partial match: + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 100.0, 'f1': 50.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [CUAD dataset](https://huggingface.co/datasets/cuad). The limitations of the biases of this dataset are not discussed, but could exhibit annotation bias given the homogeneity of annotators for this dataset. + +In terms of the metric itself, the accuracy of AUPR has been debated because its estimates are quite noisy and because of the fact that reducing the Precision-Recall Curve to a single number ignores the fact that it is about the tradeoffs between the different systems or performance points plotted and not the performance of an individual system. Reporting the original F1 and exact match scores is therefore useful to ensure a more complete representation of system performance. + + +## Citation + +```bibtex +@article{hendrycks2021cuad, + title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, + author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, + journal={arXiv preprint arXiv:2103.06268}, + year={2021} +} +``` + +## Further References + +- [CUAD dataset homepage](https://www.atticusprojectai.org/cuad-v1-performance-announcements) diff --git a/ST/evaluate/metrics/cuad/app.py b/ST/evaluate/metrics/cuad/app.py new file mode 100644 index 0000000000000000000000000000000000000000..f54499b1da0d074c8f40cac9a5285c3818243f16 --- /dev/null +++ b/ST/evaluate/metrics/cuad/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("cuad") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/cuad/compute_score.py b/ST/evaluate/metrics/cuad/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..f35a086e8d957e7763113efc79c59243ca320f14 --- /dev/null +++ b/ST/evaluate/metrics/cuad/compute_score.py @@ -0,0 +1,205 @@ +""" Official evaluation script for CUAD dataset. """ + +import argparse +import json +import re +import string +import sys + +import numpy as np + + +IOU_THRESH = 0.5 + + +def get_jaccard(prediction, ground_truth): + remove_tokens = [".", ",", ";", ":"] + + for token in remove_tokens: + ground_truth = ground_truth.replace(token, "") + prediction = prediction.replace(token, "") + + ground_truth, prediction = ground_truth.lower(), prediction.lower() + ground_truth, prediction = ground_truth.replace("/", " "), prediction.replace("/", " ") + ground_truth, prediction = set(ground_truth.split(" ")), set(prediction.split(" ")) + + intersection = ground_truth.intersection(prediction) + union = ground_truth.union(prediction) + jaccard = len(intersection) / len(union) + return jaccard + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def compute_precision_recall(predictions, ground_truths, qa_id): + tp, fp, fn = 0, 0, 0 + + substr_ok = "Parties" in qa_id + + # first check if ground truth is empty + if len(ground_truths) == 0: + if len(predictions) > 0: + fp += len(predictions) # false positive for each one + else: + for ground_truth in ground_truths: + assert len(ground_truth) > 0 + # check if there is a match + match_found = False + for pred in predictions: + if substr_ok: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred + else: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH + if is_match: + match_found = True + + if match_found: + tp += 1 + else: + fn += 1 + + # now also get any fps by looping through preds + for pred in predictions: + # Check if there's a match. if so, don't count (don't want to double count based on the above) + # but if there's no match, then this is a false positive. + # (Note: we get the true positives in the above loop instead of this loop so that we don't double count + # multiple predictions that are matched with the same answer.) + match_found = False + for ground_truth in ground_truths: + assert len(ground_truth) > 0 + if substr_ok: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred + else: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH + if is_match: + match_found = True + + if not match_found: + fp += 1 + + precision = tp / (tp + fp) if tp + fp > 0 else np.nan + recall = tp / (tp + fn) if tp + fn > 0 else np.nan + + return precision, recall + + +def process_precisions(precisions): + """ + Processes precisions to ensure that precision and recall don't both get worse. + Assumes the list precision is sorted in order of recalls + """ + precision_best = precisions[::-1] + for i in range(1, len(precision_best)): + precision_best[i] = max(precision_best[i - 1], precision_best[i]) + precisions = precision_best[::-1] + return precisions + + +def get_aupr(precisions, recalls): + processed_precisions = process_precisions(precisions) + aupr = np.trapz(processed_precisions, recalls) + if np.isnan(aupr): + return 0 + return aupr + + +def get_prec_at_recall(precisions, recalls, recall_thresh): + """Assumes recalls are sorted in increasing order""" + processed_precisions = process_precisions(precisions) + prec_at_recall = 0 + for prec, recall in zip(processed_precisions, recalls): + if recall >= recall_thresh: + prec_at_recall = prec + break + return prec_at_recall + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, predictions, ground_truths): + score = 0 + for pred in predictions: + for ground_truth in ground_truths: + score = metric_fn(pred, ground_truth) + if score == 1: # break the loop when one prediction matches the ground truth + break + if score == 1: + break + return score + + +def compute_score(dataset, predictions): + f1 = exact_match = total = 0 + precisions = [] + recalls = [] + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + print(message, file=sys.stderr) + continue + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + precision, recall = compute_precision_recall(prediction, ground_truths, qa["id"]) + + precisions.append(precision) + recalls.append(recall) + + if precision == 0 and recall == 0: + f1 += 0 + else: + f1 += 2 * (precision * recall) / (precision + recall) + + exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + + precisions = [x for _, x in sorted(zip(recalls, precisions))] + recalls.sort() + + f1 = 100.0 * f1 / total + exact_match = 100.0 * exact_match / total + aupr = get_aupr(precisions, recalls) + + prec_at_90_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.9) + prec_at_80_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.8) + + return { + "exact_match": exact_match, + "f1": f1, + "aupr": aupr, + "prec_at_80_recall": prec_at_80_recall, + "prec_at_90_recall": prec_at_90_recall, + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Evaluation for CUAD") + parser.add_argument("dataset_file", help="Dataset file") + parser.add_argument("prediction_file", help="Prediction File") + args = parser.parse_args() + with open(args.dataset_file) as dataset_file: + dataset_json = json.load(dataset_file) + dataset = dataset_json["data"] + with open(args.prediction_file) as prediction_file: + predictions = json.load(prediction_file) + print(json.dumps(compute_score(dataset, predictions))) diff --git a/ST/evaluate/metrics/cuad/cuad.py b/ST/evaluate/metrics/cuad/cuad.py new file mode 100644 index 0000000000000000000000000000000000000000..21aec5af413a5e8321704ca6ec172df48a900ab2 --- /dev/null +++ b/ST/evaluate/metrics/cuad/cuad.py @@ -0,0 +1,117 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CUAD metric. """ + +import datasets + +import evaluate + +from .compute_score import compute_score + + +_CITATION = """\ +@article{hendrycks2021cuad, + title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, + author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, + journal={arXiv preprint arXiv:2103.06268}, + year={2021} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 1 of the Contract +Understanding Atticus Dataset (CUAD). +Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510 +commercial legal contracts that have been manually labeled to identify 41 categories of important +clauses that lawyers look for when reviewing contracts in connection with corporate transactions. +""" + +_KWARGS_DESCRIPTION = """ +Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall). +Args: + predictions: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair as given in the references (see below) + - 'prediction_text': list of possible texts for the answer, as a list of strings + depending on a threshold on the confidence probability of each prediction. + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a Dict in the CUAD dataset format + { + 'text': list of possible texts for the answer, as a list of strings + 'answer_start': list of start positions for the answer, as a list of ints + } + Note that answer_start values are not taken into account to compute the metric. +Returns: + 'exact_match': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer + 'aupr': Area Under the Precision-Recall curve + 'prec_at_80_recall': Precision at 80% recall + 'prec_at_90_recall': Precision at 90% recall +Examples: + >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] + >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] + >>> cuad_metric = evaluate.load("cuad") + >>> results = cuad_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CUAD(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "id": datasets.Value("string"), + "prediction_text": datasets.features.Sequence(datasets.Value("string")), + }, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + { + "text": datasets.Value("string"), + "answer_start": datasets.Value("int32"), + } + ), + }, + } + ), + codebase_urls=["https://www.atticusprojectai.org/cuad"], + reference_urls=["https://www.atticusprojectai.org/cuad"], + ) + + def _compute(self, predictions, references): + pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} + dataset = [ + { + "paragraphs": [ + { + "qas": [ + { + "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], + "id": ref["id"], + } + for ref in references + ] + } + ] + } + ] + score = compute_score(dataset=dataset, predictions=pred_dict) + return score diff --git a/ST/evaluate/metrics/cuad/requirements.txt b/ST/evaluate/metrics/cuad/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/cuad/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/exact_match/README.md b/ST/evaluate/metrics/exact_match/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21dc1b801a3c6691d3c234b57fb94b12b416ad0b --- /dev/null +++ b/ST/evaluate/metrics/exact_match/README.md @@ -0,0 +1,119 @@ +--- +title: Exact Match +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list. +--- + +# Metric Card for Exact Match + + +## Metric Description +A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. + +- **Example 1**: The exact match score of prediction "Happy Birthday!" is 0, given its reference is "Happy New Year!". +- **Example 2**: The exact match score of prediction "The Colour of Magic (1983)" is 1, given its reference is also "The Colour of Magic (1983)". + +The exact match score of a set of predictions is the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. + +- **Example**: The exact match score of the set {Example 1, Example 2} (above) is 0.5. + + +## How to Use +At minimum, this metric takes as input predictions and references: +```python +>>> from evaluate import load +>>> exact_match_metric = load("exact_match") +>>> results = exact_match_metric.compute(predictions=predictions, references=references) +``` + +### Inputs +- **`predictions`** (`list` of `str`): List of predicted texts. +- **`references`** (`list` of `str`): List of reference texts. +- **`regexes_to_ignore`** (`list` of `str`): Regex expressions of characters to ignore when calculating the exact matches. Defaults to `None`. Note: the regex changes are applied before capitalization is normalized. +- **`ignore_case`** (`bool`): If `True`, turns everything to lowercase so that capitalization differences are ignored. Defaults to `False`. +- **`ignore_punctuation`** (`bool`): If `True`, removes punctuation before comparing strings. Defaults to `False`. +- **`ignore_numbers`** (`bool`): If `True`, removes all digits before comparing strings. Defaults to `False`. + + +### Output Values +This metric outputs a dictionary with one value: the average exact match score. + +```python +{'exact_match': 1.0} +``` + +This metric's range is 0-1, inclusive. Here, 0.0 means no prediction/reference pairs were matches, while 1.0 means they all were. + +#### Values from Popular Papers +The exact match metric is often included in other metrics, such as SQuAD. For example, the [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an Exact Match score of 40.0%. They also report that the human performance Exact Match score on the dataset was 80.3%. + +### Examples +Without including any regexes to ignore: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds) +>>> print(round(results["exact_match"], 2)) +0.25 +``` + +Ignoring regexes "the" and "yell", as well as ignoring case and punctuation: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) +>>> print(round(results["exact_match"], 2)) +0.5 +``` +Note that in the example above, because the regexes are ignored before the case is normalized, "yell" from "YELLING" is not deleted. + +Ignoring "the", "yell", and "YELL", as well as ignoring case and punctuation: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) +>>> print(round(results["exact_match"], 2)) +0.75 +``` + +Ignoring "the", "yell", and "YELL", as well as ignoring case, punctuation, and numbers: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) +>>> print(round(results["exact_match"], 2)) +1.0 +``` + +An example that includes sentences: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] +>>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] +>>> results = exact_match.compute(references=refs, predictions=preds) +>>> print(round(results["exact_match"], 2)) +0.33 +``` + + +## Limitations and Bias +This metric is limited in that it outputs the same score for something that is completely wrong as for something that is correct except for a single character. In other words, there is no award for being *almost* right. + +## Citation + +## Further References +- Also used in the [SQuAD metric](https://github.com/huggingface/datasets/tree/master/metrics/squad) diff --git a/ST/evaluate/metrics/exact_match/app.py b/ST/evaluate/metrics/exact_match/app.py new file mode 100644 index 0000000000000000000000000000000000000000..dd86c1cb619adb86db92ee10c5e115e64d987f04 --- /dev/null +++ b/ST/evaluate/metrics/exact_match/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("exact_match") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/exact_match/exact_match.py b/ST/evaluate/metrics/exact_match/exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..d8c499b3722b0bdbbf3d8a7e3d48899513f27d19 --- /dev/null +++ b/ST/evaluate/metrics/exact_match/exact_match.py @@ -0,0 +1,136 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exact Match metric.""" +import re +import string + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions: List of predicted texts. + references: List of reference texts. + regexes_to_ignore: List, defaults to None. Regex expressions of characters to + ignore when calculating the exact matches. Note: these regexes are removed + from the input data before the changes based on the options below (e.g. ignore_case, + ignore_punctuation, ignore_numbers) are applied. + ignore_case: Boolean, defaults to False. If true, turns everything + to lowercase so that capitalization differences are ignored. + ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before + comparing predictions and references. + ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before + comparing predictions and references. +Returns: + exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 1.0, inclusive. +Examples: + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds) + >>> print(round(results["exact_match"], 2)) + 0.25 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) + >>> print(round(results["exact_match"], 2)) + 0.5 + + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) + >>> print(round(results["exact_match"], 2)) + 0.75 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) + >>> print(round(results["exact_match"], 2)) + 1.0 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] + >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] + >>> results = exact_match.compute(references=refs, predictions=preds) + >>> print(round(results["exact_match"], 2)) + 0.33 +""" + +_CITATION = """ +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ExactMatch(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + reference_urls=[], + ) + + def _compute( + self, + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, + ): + + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} diff --git a/ST/evaluate/metrics/exact_match/requirements.txt b/ST/evaluate/metrics/exact_match/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/exact_match/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/f1/README.md b/ST/evaluate/metrics/f1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8f38be6e6533b94833ca11ce198095f064b5e79b --- /dev/null +++ b/ST/evaluate/metrics/f1/README.md @@ -0,0 +1,137 @@ +--- +title: F1 +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: + F1 = 2 * (precision * recall) / (precision + recall) +--- + +# Metric Card for F1 + + +## Metric Description + +The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: +F1 = 2 * (precision * recall) / (precision + recall) + + +## How to Use + +At minimum, this metric requires predictions and references as input + +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(predictions=[0, 1], references=[0, 1]) +>>> print(results) +["{'f1': 1.0}"] +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted labels. +- **references** (`list` of `int`): Ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. + + +### Output Values +- **f1**(`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. + +Output Example(s): +```python +{'f1': 0.26666666666666666} +``` +```python +{'f1': array([0.8, 0.0, 0.0])} +``` + +This metric outputs a dictionary, with either a single f1 score, of type `float`, or an array of f1 scores, with entries of type `float`. + + +#### Values from Popular Papers + + + + +### Examples + +Example 1-A simple binary example +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) +>>> print(results) +{'f1': 0.5} +``` + +Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) +>>> print(round(results['f1'], 2)) +0.67 +``` + +Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) +>>> print(round(results['f1'], 2)) +0.35 +``` + +Example 4-A multiclass example, with different values for the `average` input. +```python +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") +>>> print(round(results['f1'], 2)) +0.27 +>>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") +>>> print(round(results['f1'], 2)) +0.33 +>>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") +>>> print(round(results['f1'], 2)) +0.27 +>>> results = f1_metric.compute(predictions=predictions, references=references, average=None) +>>> print(results) +{'f1': array([0.8, 0. , 0. ])} +``` + + +## Limitations and Bias + + + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References \ No newline at end of file diff --git a/ST/evaluate/metrics/f1/app.py b/ST/evaluate/metrics/f1/app.py new file mode 100644 index 0000000000000000000000000000000000000000..56baa5ec6c9867771ebbbf6821366b4ebdd31ef6 --- /dev/null +++ b/ST/evaluate/metrics/f1/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("f1") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/f1/f1.py b/ST/evaluate/metrics/f1/f1.py new file mode 100644 index 0000000000000000000000000000000000000000..fe768348912789ed99aa4f00e30dd222e28acaeb --- /dev/null +++ b/ST/evaluate/metrics/f1/f1.py @@ -0,0 +1,130 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""F1 metric.""" + +import datasets +from sklearn.metrics import f1_score + +import evaluate + + +_DESCRIPTION = """ +The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: +F1 = 2 * (precision * recall) / (precision + recall) +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted labels. + references (`list` of `int`): Ground truth labels. + labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. + pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. + average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). + sample_weight (`list` of `float`): Sample weights Defaults to None. + +Returns: + f1 (`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. + +Examples: + + Example 1-A simple binary example + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) + >>> print(results) + {'f1': 0.5} + + Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) + >>> print(round(results['f1'], 2)) + 0.67 + + Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) + >>> print(round(results['f1'], 2)) + 0.35 + + Example 4-A multiclass example, with different values for the `average` input. + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") + >>> print(round(results['f1'], 2)) + 0.27 + >>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") + >>> print(round(results['f1'], 2)) + 0.33 + >>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") + >>> print(round(results['f1'], 2)) + 0.27 + >>> results = f1_metric.compute(predictions=predictions, references=references, average=None) + >>> print(results) + {'f1': array([0.8, 0. , 0. ])} + + Example 5-A multi-label example + >>> f1_metric = evaluate.load("f1", "multilabel") + >>> results = f1_metric.compute(predictions=[[0, 1, 1], [1, 1, 0]], references=[[0, 1, 1], [0, 1, 0]], average="macro") + >>> print(round(results['f1'], 2)) + 0.67 +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class F1(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html"], + ) + + def _compute(self, predictions, references, labels=None, pos_label=1, average="binary", sample_weight=None): + score = f1_score( + references, predictions, labels=labels, pos_label=pos_label, average=average, sample_weight=sample_weight + ) + return {"f1": float(score) if score.size == 1 else score} diff --git a/ST/evaluate/metrics/f1/requirements.txt b/ST/evaluate/metrics/f1/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/f1/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/frugalscore/README.md b/ST/evaluate/metrics/frugalscore/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a1a05b004dd9422ce4bfaa5c4d5a2f87851b8480 --- /dev/null +++ b/ST/evaluate/metrics/frugalscore/README.md @@ -0,0 +1,116 @@ +--- +title: +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. +--- + + +## Metric Description +FrugalScore is a reference-based metric for Natural Language Generation (NLG) model evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. + +The FrugalScore models are obtained by continuing the pretraining of small models on a synthetic dataset constructed using summarization, backtranslation and denoising models. During the training, the small models learn the internal mapping of the expensive metric, including any similarity function. + +## How to use + +When loading FrugalScore, you can indicate the model you wish to use to compute the score. The default model is `moussaKam/frugalscore_tiny_bert-base_bert-score`, and a full list of models can be found in the [Limitations and bias](#Limitations-and-bias) section. + +```python +>>> frugalscore = evaluate.load("frugalscore", "moussaKam/frugalscore_medium_bert-base_mover-score") +``` + +FrugalScore calculates how good are the predictions given some references, based on a set of scores. + +The inputs it takes are: + +`predictions`: a list of strings representing the predictions to score. + +`references`: a list of string representing the references for each prediction. + +Its optional arguments are: + +`batch_size`: the batch size for predictions (default value is `32`). + +`max_length`: the maximum sequence length (default value is `128`). + +`device`: either "gpu" or "cpu" (default value is `None`). + +```python +>>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face'], batch_size=16, max_length=64, device="gpu") +``` + +## Output values + +The output of FrugalScore is a dictionary with the list of scores for each prediction-reference pair: +```python +{'scores': [0.6307541, 0.6449357]} +``` + +### Values from popular papers +The [original FrugalScore paper](https://arxiv.org/abs/2110.08559) reported that FrugalScore-Tiny retains 97.7/94.7% of the original performance compared to [BertScore](https://huggingface.co/metrics/bertscore) while running 54 times faster and having 84 times less parameters. + +## Examples + +Maximal values (exact match between `references` and `predictions`): + +```python +>>> frugalscore = evaluate.load("frugalscore") +>>> results = frugalscore.compute(predictions=['hello world'], references=['hello world']) +>>> print(results) +{'scores': [0.9891098]} +``` + +Partial values: + +```python +>>> frugalscore = evaluate.load("frugalscore") +>>> results = frugalscore.compute(predictions=['hello world'], references=['hugging face']) +>>> print(results) +{'scores': [0.42482382]} +``` + +## Limitations and bias + +FrugalScore is based on [BertScore](https://huggingface.co/metrics/bertscore) and [MoverScore](https://arxiv.org/abs/1909.02622), and the models used are based on the original models used for these scores. + +The full list of available models for FrugalScore is: + +| FrugalScore | Student | Teacher | Method | +|----------------------------------------------------|-------------|----------------|------------| +| [moussaKam/frugalscore_tiny_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_bert-score) | BERT-tiny | BERT-Base | BERTScore | +| [moussaKam/frugalscore_small_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_bert-score) | BERT-small | BERT-Base | BERTScore | +| [moussaKam/frugalscore_medium_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_bert-score) | BERT-medium | BERT-Base | BERTScore | +| [moussaKam/frugalscore_tiny_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_roberta_bert-score) | BERT-tiny | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_small_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_roberta_bert-score) | BERT-small | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_medium_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_roberta_bert-score) | BERT-medium | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_tiny_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_deberta_bert-score) | BERT-tiny | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_small_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_deberta_bert-score) | BERT-small | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_medium_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_deberta_bert-score) | BERT-medium | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_tiny_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_mover-score) | BERT-tiny | BERT-Base | MoverScore | +| [moussaKam/frugalscore_small_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_mover-score) | BERT-small | BERT-Base | MoverScore | +| [moussaKam/frugalscore_medium_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_mover-score) | BERT-medium | BERT-Base | MoverScore | + +Depending on the size of the model picked, the loading time will vary: the `tiny` models will load very quickly, whereas the `medium` ones can take several minutes, depending on your Internet connection. + +## Citation +```bibtex +@article{eddine2021frugalscore, + title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, + author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, + journal={arXiv preprint arXiv:2110.08559}, + year={2021} +} +``` + +## Further References +- [Original FrugalScore code](https://github.com/moussaKam/FrugalScore) +- [FrugalScore paper](https://arxiv.org/abs/2110.08559) diff --git a/ST/evaluate/metrics/frugalscore/app.py b/ST/evaluate/metrics/frugalscore/app.py new file mode 100644 index 0000000000000000000000000000000000000000..96ddc17c415bf2e6c362fdbebbd7da7fe48ea69f --- /dev/null +++ b/ST/evaluate/metrics/frugalscore/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("frugalscore") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/frugalscore/frugalscore.py b/ST/evaluate/metrics/frugalscore/frugalscore.py new file mode 100644 index 0000000000000000000000000000000000000000..b6f7ee5f3ff9926b9a6c7eb5a2f60eab19b0e2fc --- /dev/null +++ b/ST/evaluate/metrics/frugalscore/frugalscore.py @@ -0,0 +1,117 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current metric script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""FrugalScore metric.""" + +import datasets +import torch +from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments + +import evaluate + + +_CITATION = """\ +@article{eddine2021frugalscore, + title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, + author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, + journal={arXiv preprint arXiv:2110.08559}, + year={2021} +} +""" + +_DESCRIPTION = """\ +FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. +""" + + +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores. +Args: + predictions (list of str): list of predictions to score. Each predictions + should be a string. + references (list of str): list of reference for each prediction. Each + reference should be a string. + batch_size (int): the batch size for predictions. + max_length (int): maximum sequence length. + device (str): either gpu or cpu +Returns: + scores (list of int): list of scores. +Examples: + >>> frugalscore = evaluate.load("frugalscore") + >>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face']) + >>> print([round(s, 3) for s in results["scores"]]) + [0.631, 0.645] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class FRUGALSCORE(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + "references": datasets.Value("string"), + } + ), + homepage="https://github.com/moussaKam/FrugalScore", + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + checkpoint = "moussaKam/frugalscore_tiny_bert-base_bert-score" + else: + checkpoint = self.config_name + self.model = AutoModelForSequenceClassification.from_pretrained(checkpoint) + self.tokenizer = AutoTokenizer.from_pretrained(checkpoint) + + def _compute( + self, + predictions, + references, + batch_size=32, + max_length=128, + device=None, + ): + """Returns the scores""" + assert len(predictions) == len( + references + ), "predictions and references should have the same number of sentences." + if device is not None: + assert device in ["gpu", "cpu"], "device should be either gpu or cpu." + else: + device = "gpu" if torch.cuda.is_available() else "cpu" + training_args = TrainingArguments( + "trainer", + fp16=(device == "gpu"), + per_device_eval_batch_size=batch_size, + report_to="all", + no_cuda=(device == "cpu"), + log_level="warning", + ) + dataset = {"sentence1": predictions, "sentence2": references} + raw_datasets = datasets.Dataset.from_dict(dataset) + + def tokenize_function(data): + return self.tokenizer( + data["sentence1"], data["sentence2"], max_length=max_length, truncation=True, padding=True + ) + + tokenized_datasets = raw_datasets.map(tokenize_function, batched=True) + tokenized_datasets.remove_columns(["sentence1", "sentence2"]) + trainer = Trainer(self.model, training_args, tokenizer=self.tokenizer) + predictions = trainer.predict(tokenized_datasets) + return {"scores": list(predictions.predictions.squeeze(-1))} diff --git a/ST/evaluate/metrics/frugalscore/requirements.txt b/ST/evaluate/metrics/frugalscore/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..84eb91d19f4a5974796fd201cc805cbb22c907bb --- /dev/null +++ b/ST/evaluate/metrics/frugalscore/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +transformers \ No newline at end of file diff --git a/ST/evaluate/metrics/glue/README.md b/ST/evaluate/metrics/glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..af78d504d9d9840da469c18d5e2af798d078472b --- /dev/null +++ b/ST/evaluate/metrics/glue/README.md @@ -0,0 +1,123 @@ +--- +title: GLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + GLUE, the General Language Understanding Evaluation benchmark + (https://gluebenchmark.com/) is a collection of resources for training, + evaluating, and analyzing natural language understanding systems. +--- + +# Metric Card for GLUE + +## Metric description +This metric is used to compute the GLUE evaluation metric associated to each [GLUE dataset](https://huggingface.co/datasets/glue). + +GLUE, the General Language Understanding Evaluation benchmark is a collection of resources for training, evaluating, and analyzing natural language understanding systems. + +## How to use + +There are two steps: (1) loading the GLUE metric relevant to the subset of the GLUE dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant GLUE metric** : the subsets of GLUE are the following: `sst2`, `mnli`, `mnli_mismatched`, `mnli_matched`, `qnli`, `rte`, `wnli`, `cola`,`stsb`, `mrpc`, `qqp`, and `hans`. + +More information about the different subsets of the GLUE dataset can be found on the [GLUE dataset page](https://huggingface.co/datasets/glue). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation. + +```python +from evaluate import load +glue_metric = load('glue', 'sst2') +references = [0, 1] +predictions = [0, 1] +results = glue_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the metric depends on the GLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`pearson`: a measure of the linear relationship between two datasets (see [Pearson correlation](https://huggingface.co/metrics/pearsonr) for more information). Its range is between -1 and +1, with 0 implying no correlation, and -1/+1 implying an exact linear relationship. Positive correlations imply that as x increases, so does y, whereas negative correlations imply that as x increases, y decreases. + +`spearmanr`: a nonparametric measure of the monotonicity of the relationship between two datasets(see [Spearman Correlation](https://huggingface.co/metrics/spearmanr) for more information). `spearmanr` has the same range as `pearson`. + +`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. + +The `cola` subset returns `matthews_correlation`, the `stsb` subset returns `pearson` and `spearmanr`, the `mrpc` and `qqp` subsets return both `accuracy` and `f1`, and all other subsets of GLUE return only accuracy. + +### Values from popular papers +The [original GLUE paper](https://huggingface.co/datasets/glue) reported average scores ranging from 58 to 64%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/glue). + +## Examples + +Maximal values for the MRPC subset (which outputs `accuracy` and `f1`): + +```python +from evaluate import load +glue_metric = load('glue', 'mrpc') # 'mrpc' or 'qqp' +references = [0, 1] +predictions = [0, 1] +results = glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0, 'f1': 1.0} +``` + +Minimal values for the STSB subset (which outputs `pearson` and `spearmanr`): + +```python +from evaluate import load +glue_metric = load('glue', 'stsb') +references = [0., 1., 2., 3., 4., 5.] +predictions = [-10., -11., -12., -13., -14., -15.] +results = glue_metric.compute(predictions=predictions, references=references) +print(results) +{'pearson': -1.0, 'spearmanr': -1.0} +``` + +Partial match for the COLA subset (which outputs `matthews_correlation`) + +```python +from evaluate import load +glue_metric = load('glue', 'cola') +references = [0, 1] +predictions = [1, 1] +results = glue_metric.compute(predictions=predictions, references=references) +results +{'matthews_correlation': 0.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [GLUE dataset](https://huggingface.co/datasets/glue). + +While the GLUE dataset is meant to represent "General Language Understanding", the tasks represented in it are not necessarily representative of language understanding, and should not be interpreted as such. + +Also, while the GLUE subtasks were considered challenging during its creation in 2019, they are no longer considered as such given the impressive progress made since then. A more complex (or "stickier") version of it, called [SuperGLUE](https://huggingface.co/datasets/super_glue), was subsequently created. + +## Citation + +```bibtex + @inproceedings{wang2019glue, + title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, + author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, + note={In the Proceedings of ICLR.}, + year={2019} +} +``` + +## Further References + +- [GLUE benchmark homepage](https://gluebenchmark.com/) +- [Fine-tuning a model with the Trainer API](https://huggingface.co/course/chapter3/3?) diff --git a/ST/evaluate/metrics/glue/app.py b/ST/evaluate/metrics/glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..41b7d78d34856615fee2762cd51aacecc1cdc910 --- /dev/null +++ b/ST/evaluate/metrics/glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("glue", "sst2") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/glue/glue.py b/ST/evaluate/metrics/glue/glue.py new file mode 100644 index 0000000000000000000000000000000000000000..8c607bf1e2fef42fa3abcf1c672c90b576e6eef0 --- /dev/null +++ b/ST/evaluate/metrics/glue/glue.py @@ -0,0 +1,156 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" GLUE benchmark metric. """ + +import datasets +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import f1_score, matthews_corrcoef + +import evaluate + + +_CITATION = """\ +@inproceedings{wang2019glue, + title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, + author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, + note={In the Proceedings of ICLR.}, + year={2019} +} +""" + +_DESCRIPTION = """\ +GLUE, the General Language Understanding Evaluation benchmark +(https://gluebenchmark.com/) is a collection of resources for training, +evaluating, and analyzing natural language understanding systems. +""" + +_KWARGS_DESCRIPTION = """ +Compute GLUE evaluation metric associated to each GLUE dataset. +Args: + predictions: list of predictions to score. + Each translation should be tokenized into a list of tokens. + references: list of lists of references for each translation. + Each reference should be tokenized into a list of tokens. +Returns: depending on the GLUE subset, one or several of: + "accuracy": Accuracy + "f1": F1 score + "pearson": Pearson Correlation + "spearmanr": Spearman Correlation + "matthews_correlation": Matthew Correlation +Examples: + + >>> glue_metric = evaluate.load('glue', 'sst2') # 'sst2' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"] + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> glue_metric = evaluate.load('glue', 'mrpc') # 'mrpc' or 'qqp' + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> glue_metric = evaluate.load('glue', 'stsb') + >>> references = [0., 1., 2., 3., 4., 5.] + >>> predictions = [0., 1., 2., 3., 4., 5.] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)}) + {'pearson': 1.0, 'spearmanr': 1.0} + + >>> glue_metric = evaluate.load('glue', 'cola') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'matthews_correlation': 1.0} +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds)) + return { + "accuracy": acc, + "f1": f1, + } + + +def pearson_and_spearman(preds, labels): + pearson_corr = float(pearsonr(preds, labels)[0]) + spearman_corr = float(spearmanr(preds, labels)[0]) + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + } + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Glue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "sst2", + "mnli", + "mnli_mismatched", + "mnli_matched", + "cola", + "stsb", + "mrpc", + "qqp", + "qnli", + "rte", + "wnli", + "hans", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' + '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64" if self.config_name != "stsb" else "float32"), + "references": datasets.Value("int64" if self.config_name != "stsb" else "float32"), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references): + if self.config_name == "cola": + return {"matthews_correlation": matthews_corrcoef(references, predictions)} + elif self.config_name == "stsb": + return pearson_and_spearman(predictions, references) + elif self.config_name in ["mrpc", "qqp"]: + return acc_and_f1(predictions, references) + elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' + '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' + ) diff --git a/ST/evaluate/metrics/glue/requirements.txt b/ST/evaluate/metrics/glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..56f7d3a98bb1bcfd747153724d77cff546124757 --- /dev/null +++ b/ST/evaluate/metrics/glue/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/google_bleu/README.md b/ST/evaluate/metrics/google_bleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4d4814295cbcb796bc000d0db950ca9b35bc1d73 --- /dev/null +++ b/ST/evaluate/metrics/google_bleu/README.md @@ -0,0 +1,153 @@ +--- +title: Google BLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The BLEU score has some undesirable properties when used for single + sentences, as it was designed to be a corpus measure. We therefore + use a slightly different score for our RL experiments which we call + the 'GLEU score'. For the GLEU score, we record all sub-sequences of + 1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then + compute a recall, which is the ratio of the number of matching n-grams + to the number of total n-grams in the target (ground truth) sequence, + and a precision, which is the ratio of the number of matching n-grams + to the number of total n-grams in the generated output sequence. Then + GLEU score is simply the minimum of recall and precision. This GLEU + score's range is always between 0 (no matches) and 1 (all match) and + it is symmetrical when switching output and target. According to + our experiments, GLEU score correlates quite well with the BLEU + metric on a corpus level but does not have its drawbacks for our per + sentence reward objective. +--- + +# Metric Card for Google BLEU + + +## Metric Description +The BLEU score has some undesirable properties when used for single sentences, as it was designed to be a corpus measure. The Google BLEU score is designed to limit these undesirable properties when used for single sentences. + +To calculate this score, all sub-sequences of 1, 2, 3 or 4 tokens in output and target sequence (n-grams) are recorded. The precision and recall, described below, are then computed. + +- **precision:** the ratio of the number of matching n-grams to the number of total n-grams in the generated output sequence +- **recall:** the ratio of the number of matching n-grams to the number of total n-grams in the target (ground truth) sequence + +The minimum value of precision and recall is then returned as the score. + + +## Intended Uses +This metric is generally used to evaluate machine translation models. It is especially used when scores of individual (prediction, reference) sentence pairs are needed, as opposed to when averaging over the (prediction, reference) scores for a whole corpus. That being said, it can also be used when averaging over the scores for a whole corpus. + +Because it performs better on individual sentence pairs as compared to BLEU, Google BLEU has also been used in RL experiments. + +## How to Use +This metric takes a list of predicted sentences, as well as a list of references. + +```python +sentence1 = "the cat sat on the mat" +sentence2 = "the cat ate the mat" +google_bleu = evaluate.load("google_bleu") +result = google_bleu.compute(predictions=[sentence1], references=[[sentence2]]) +print(result) +>>> {'google_bleu': 0.3333333333333333} +``` + +### Inputs +- **predictions** (list of str): list of translations to score. +- **references** (list of list of str): list of lists of references for each translation. +- **tokenizer** : approach used for tokenizing `predictions` and `references`. +The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. This can be replaced by any function that takes a string as input and returns a list of tokens as output. +- **min_len** (int): The minimum order of n-gram this function should extract. Defaults to 1. +- **max_len** (int): The maximum order of n-gram this function should extract. Defaults to 4. + +### Output Values +This metric returns the following in a dict: +- **google_bleu** (float): google_bleu score + +The output format is as follows: +```python +{'google_bleu': google_bleu score} +``` + +This metric can take on values from 0 to 1, inclusive. Higher scores are better, with 0 indicating no matches, and 1 indicating a perfect match. + +Note that this score is symmetrical when switching output and target. This means that, given two sentences, `sentence1` and `sentence2`, whatever score is output when `sentence1` is the predicted sentence and `sencence2` is the reference sentence will be the same as when the sentences are swapped and `sentence2` is the predicted sentence while `sentence1` is the reference sentence. In code, this looks like: + +```python +predictions = "the cat sat on the mat" +references = "the cat ate the mat" +google_bleu = evaluate.load("google_bleu") +result_a = google_bleu.compute(predictions=[predictions], references=[[references]]) +result_b = google_bleu.compute(predictions=[predictions], references=[[references]]) +print(result_a == result_b) +>>> True +``` + +#### Values from Popular Papers + + +### Examples +Example with one reference per sample: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references) +>>> print(round(results["google_bleu"], 2)) +0.44 +``` + +Example with multiple references for the first sample: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references) +>>> print(round(results["google_bleu"], 2)) +0.61 +``` + +Example with multiple references for the first sample, and with `min_len` adjusted to `2`, instead of the default `1`, which means that the function extracts n-grams of length `2`: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references, min_len=2) +>>> print(round(results["google_bleu"], 2)) +0.53 +``` + +Example with multiple references for the first sample, with `min_len` adjusted to `2`, instead of the default `1`, and `max_len` adjusted to `6` instead of the default `4`: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions,references=references, min_len=2, max_len=6) +>>> print(round(results["google_bleu"], 2)) +0.4 +``` + +## Limitations and Bias + +The GoogleBLEU metric does not come with a predefined tokenization function; previous versions simply used `split()` to split the input strings into tokens. Using a tokenizer such as the default one, `tokenizer_13a`, makes results more standardized and reproducible. The BLEU and sacreBLEU metrics also use this default tokenizer. + +## Citation +```bibtex +@misc{wu2016googles, +title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, +author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes and Jeffrey Dean}, +year={2016}, +eprint={1609.08144}, +archivePrefix={arXiv}, +primaryClass={cs.CL} +} +``` +## Further References +- This Hugging Face implementation uses the [nltk.translate.gleu_score implementation](https://www.nltk.org/_modules/nltk/translate/gleu_score.html) diff --git a/ST/evaluate/metrics/google_bleu/app.py b/ST/evaluate/metrics/google_bleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..582df789b608693d0262b7c546e8aa94595b9b78 --- /dev/null +++ b/ST/evaluate/metrics/google_bleu/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("google_bleu") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/google_bleu/google_bleu.py b/ST/evaluate/metrics/google_bleu/google_bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..adcc0a31f382e65d00d7f7a7169aef20205bb577 --- /dev/null +++ b/ST/evaluate/metrics/google_bleu/google_bleu.py @@ -0,0 +1,168 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Google BLEU (aka GLEU) metric. """ + +from typing import Dict, List + +import datasets +from nltk.translate import gleu_score + +import evaluate +from evaluate import MetricInfo + +from .tokenizer_13a import Tokenizer13a + + +_CITATION = """\ +@misc{wu2016googles, + title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, + author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey + and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin + Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto + Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and + Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes + and Jeffrey Dean}, + year={2016}, + eprint={1609.08144}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +The BLEU score has some undesirable properties when used for single +sentences, as it was designed to be a corpus measure. We therefore +use a slightly different score for our RL experiments which we call +the 'GLEU score'. For the GLEU score, we record all sub-sequences of +1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then +compute a recall, which is the ratio of the number of matching n-grams +to the number of total n-grams in the target (ground truth) sequence, +and a precision, which is the ratio of the number of matching n-grams +to the number of total n-grams in the generated output sequence. Then +GLEU score is simply the minimum of recall and precision. This GLEU +score's range is always between 0 (no matches) and 1 (all match) and +it is symmetrical when switching output and target. According to +our experiments, GLEU score correlates quite well with the BLEU +metric on a corpus level but does not have its drawbacks for our per +sentence reward objective. +""" + +_KWARGS_DESCRIPTION = """\ +Computes corpus-level Google BLEU (GLEU) score of translated segments against one or more references. +Instead of averaging the sentence level GLEU scores (i.e. macro-average precision), Wu et al. (2016) sum up the matching +tokens and the max of hypothesis and reference tokens for each sentence, then compute using the aggregate values. + +Args: + predictions (list of str): list of translations to score. + references (list of list of str): list of lists of references for each translation. + tokenizer : approach used for tokenizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + min_len (int): The minimum order of n-gram this function should extract. Defaults to 1. + max_len (int): The maximum order of n-gram this function should extract. Defaults to 4. + +Returns: + 'google_bleu': google_bleu score + +Examples: + Example 1: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references) + >>> print(round(results["google_bleu"], 2)) + 0.44 + + Example 2: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references) + >>> print(round(results["google_bleu"], 2)) + 0.61 + + Example 3: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references, min_len=2) + >>> print(round(results["google_bleu"], 2)) + 0.53 + + Example 4: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions,references=references, min_len=2, max_len=6) + >>> print(round(results["google_bleu"], 2)) + 0.4 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class GoogleBleu(evaluate.Metric): + def _info(self) -> MetricInfo: + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + ) + + def _compute( + self, + predictions: List[str], + references: List[List[str]], + tokenizer=Tokenizer13a(), + min_len: int = 1, + max_len: int = 4, + ) -> Dict[str, float]: + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references = [[tokenizer(r) for r in ref] for ref in references] + predictions = [tokenizer(p) for p in predictions] + return { + "google_bleu": gleu_score.corpus_gleu( + list_of_references=references, hypotheses=predictions, min_len=min_len, max_len=max_len + ) + } diff --git a/ST/evaluate/metrics/google_bleu/requirements.txt b/ST/evaluate/metrics/google_bleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5904a71469bea16b7ccb3b41e9ed19663d20599 --- /dev/null +++ b/ST/evaluate/metrics/google_bleu/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk \ No newline at end of file diff --git a/ST/evaluate/metrics/google_bleu/tokenizer_13a.py b/ST/evaluate/metrics/google_bleu/tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..c7a1b3dbd39d7db5ed0c4d000d6d648e2cbf0009 --- /dev/null +++ b/ST/evaluate/metrics/google_bleu/tokenizer_13a.py @@ -0,0 +1,100 @@ +# Source: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_13a.py +# Copyright 2020 SacreBLEU Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from functools import lru_cache + + +class BaseTokenizer: + """A base dummy tokenizer to derive from.""" + + def signature(self): + """ + Returns a signature for the tokenizer. + :return: signature string + """ + return "none" + + def __call__(self, line): + """ + Tokenizes an input line with the tokenizer. + :param line: a segment to tokenize + :return: the tokenized line + """ + return line + + +class TokenizerRegexp(BaseTokenizer): + def signature(self): + return "re" + + def __init__(self): + self._re = [ + # language-dependent part (assuming Western languages) + (re.compile(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])"), r" \1 "), + # tokenize period and comma unless preceded by a digit + (re.compile(r"([^0-9])([\.,])"), r"\1 \2 "), + # tokenize period and comma unless followed by a digit + (re.compile(r"([\.,])([^0-9])"), r" \1 \2"), + # tokenize dash when preceded by a digit + (re.compile(r"([0-9])(-)"), r"\1 \2 "), + # one space only between words + # NOTE: Doing this in Python (below) is faster + # (re.compile(r'\s+'), r' '), + ] + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Common post-processing tokenizer for `13a` and `zh` tokenizers. + :param line: a segment to tokenize + :return: the tokenized line + """ + for (_re, repl) in self._re: + line = _re.sub(repl, line) + + # no leading or trailing spaces, single space within words + # return ' '.join(line.split()) + # This line is changed with regards to the original tokenizer (seen above) to return individual words + return line.split() + + +class Tokenizer13a(BaseTokenizer): + def signature(self): + return "13a" + + def __init__(self): + self._post_tokenizer = TokenizerRegexp() + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Tokenizes an input line using a relatively minimal tokenization + that is however equivalent to mteval-v13a, used by WMT. + + :param line: a segment to tokenize + :return: the tokenized line + """ + + # language-independent part: + line = line.replace("", "") + line = line.replace("-\n", "") + line = line.replace("\n", " ") + + if "&" in line: + line = line.replace(""", '"') + line = line.replace("&", "&") + line = line.replace("<", "<") + line = line.replace(">", ">") + + return self._post_tokenizer(f" {line} ") diff --git a/ST/evaluate/metrics/indic_glue/README.md b/ST/evaluate/metrics/indic_glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a58c8d1dcd5dcf9f60393aad3ee64e4e66c4396a --- /dev/null +++ b/ST/evaluate/metrics/indic_glue/README.md @@ -0,0 +1,108 @@ +--- +title: IndicGLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide + variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te. +--- + +# Metric Card for IndicGLUE + +## Metric description +This metric is used to compute the evaluation metric for the [IndicGLUE dataset](https://huggingface.co/datasets/indic_glue). + +IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide variety of tasks and covers 11 major Indian languages - Assamese (`as`), Bengali (`bn`), Gujarati (`gu`), Hindi (`hi`), Kannada (`kn`), Malayalam (`ml`), Marathi(`mr`), Oriya(`or`), Panjabi (`pa`), Tamil(`ta`) and Telugu (`te`). + +## How to use + +There are two steps: (1) loading the IndicGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant IndicGLUE metric** : the subsets of IndicGLUE are the following: `wnli`, `copa`, `sna`, `csqa`, `wstp`, `inltkh`, `bbca`, `cvit-mkb-clsr`, `iitp-mr`, `iitp-pr`, `actsa-sc`, `md`, and`wiki-ner`. + +More information about the different subsets of the Indic GLUE dataset can be found on the [IndicGLUE dataset page](https://indicnlp.ai4bharat.org/indic-glue/). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation for all subsets of the dataset except for `cvit-mkb-clsr`, where each prediction and reference is a vector of floats. + +```python +indic_glue_metric = evaluate.load('indic_glue', 'wnli') +references = [0, 1] +predictions = [0, 1] +results = indic_glue_metric.compute(predictions=predictions, references=references) +``` + +## Output values + +The output of the metric depends on the IndicGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`precision@10`: the fraction of the true examples among the top 10 predicted examples, with a range between 0 and 1 (see [precision](https://huggingface.co/metrics/precision) for more information). + +The `cvit-mkb-clsr` subset returns `precision@10`, the `wiki-ner` subset returns `accuracy` and `f1`, and all other subsets of Indic GLUE return only accuracy. + +### Values from popular papers + +The [original IndicGlue paper](https://aclanthology.org/2020.findings-emnlp.445.pdf) reported an average accuracy of 0.766 on the dataset, which varies depending on the subset selected. + +## Examples + +Maximal values for the WNLI subset (which outputs `accuracy`): + +```python +indic_glue_metric = evaluate.load('indic_glue', 'wnli') +references = [0, 1] +predictions = [0, 1] +results = indic_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0} +``` + +Minimal values for the Wiki-NER subset (which outputs `accuracy` and `f1`): + +```python +>>> indic_glue_metric = evaluate.load('indic_glue', 'wiki-ner') +>>> references = [0, 1] +>>> predictions = [1,0] +>>> results = indic_glue_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 1.0, 'f1': 1.0} +``` + +Partial match for the CVIT-Mann Ki Baat subset (which outputs `precision@10`) + +```python +>>> indic_glue_metric = evaluate.load('indic_glue', 'cvit-mkb-clsr') +>>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] +>>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] +>>> results = indic_glue_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'precision@10': 1.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [IndicGLUE dataset](https://huggingface.co/datasets/glue). + +## Citation + +```bibtex + @inproceedings{kakwani2020indicnlpsuite, + title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, + author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, + year={2020}, + booktitle={Findings of EMNLP}, +} +``` + +## Further References +- [IndicNLP website](https://indicnlp.ai4bharat.org/home/) diff --git a/ST/evaluate/metrics/indic_glue/app.py b/ST/evaluate/metrics/indic_glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1edafea04b601ad33ecfae98fc01dac61fabc202 --- /dev/null +++ b/ST/evaluate/metrics/indic_glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("indic_glue", "wnli") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/indic_glue/indic_glue.py b/ST/evaluate/metrics/indic_glue/indic_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..03afd1700180ad1d0eeb0d8e885c9ad4c67489e2 --- /dev/null +++ b/ST/evaluate/metrics/indic_glue/indic_glue.py @@ -0,0 +1,173 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" IndicGLUE benchmark metric. """ + +import datasets +import numpy as np +from scipy.spatial.distance import cdist +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import f1_score + +import evaluate + + +_CITATION = """\ + @inproceedings{kakwani2020indicnlpsuite, + title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, + author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, + year={2020}, + booktitle={Findings of EMNLP}, +} +""" + +_DESCRIPTION = """\ + IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide + variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te. +""" + +_KWARGS_DESCRIPTION = """ +Compute IndicGLUE evaluation metric associated to each IndicGLUE dataset. +Args: + predictions: list of predictions to score (as int64), + except for 'cvit-mkb-clsr' where each prediction is a vector (of float32). + references: list of ground truth labels corresponding to the predictions (as int64), + except for 'cvit-mkb-clsr' where each reference is a vector (of float32). +Returns: depending on the IndicGLUE subset, one or several of: + "accuracy": Accuracy + "f1": F1 score + "precision": Precision@10 +Examples: + + >>> indic_glue_metric = evaluate.load('indic_glue', 'wnli') # 'wnli' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"] + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> indic_glue_metric = evaluate.load('indic_glue', 'wiki-ner') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> indic_glue_metric = evaluate.load('indic_glue', 'cvit-mkb-clsr') + >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] + >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'precision@10': 1.0} + +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds)) + return { + "accuracy": acc, + "f1": f1, + } + + +def precision_at_10(en_sentvecs, in_sentvecs): + en_sentvecs = np.array(en_sentvecs) + in_sentvecs = np.array(in_sentvecs) + n = en_sentvecs.shape[0] + + # mean centering + en_sentvecs = en_sentvecs - np.mean(en_sentvecs, axis=0) + in_sentvecs = in_sentvecs - np.mean(in_sentvecs, axis=0) + + sim = cdist(en_sentvecs, in_sentvecs, "cosine") + actual = np.array(range(n)) + preds = sim.argsort(axis=1)[:, :10] + matches = np.any(preds == actual[:, None], axis=1) + return float(matches.mean()) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class IndicGlue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "wnli", + "copa", + "sna", + "csqa", + "wstp", + "inltkh", + "bbca", + "cvit-mkb-clsr", + "iitp-mr", + "iitp-pr", + "actsa-sc", + "md", + "wiki-ner", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' + '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' + '"wiki-ner"]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")), + "references": datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy" if self.config_name != "cvit-mkb-clsr" else None, + ) + + def _compute(self, predictions, references): + if self.config_name == "cvit-mkb-clsr": + return {"precision@10": precision_at_10(predictions, references)} + elif self.config_name in ["wiki-ner"]: + return acc_and_f1(predictions, references) + elif self.config_name in [ + "wnli", + "copa", + "sna", + "csqa", + "wstp", + "inltkh", + "bbca", + "iitp-mr", + "iitp-pr", + "actsa-sc", + "md", + ]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' + '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' + '"wiki-ner"]' + ) diff --git a/ST/evaluate/metrics/indic_glue/requirements.txt b/ST/evaluate/metrics/indic_glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..56f7d3a98bb1bcfd747153724d77cff546124757 --- /dev/null +++ b/ST/evaluate/metrics/indic_glue/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/mae/README.md b/ST/evaluate/metrics/mae/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe238e1f67e2769a8ad9d215fb7fa910aa5d9eb7 --- /dev/null +++ b/ST/evaluate/metrics/mae/README.md @@ -0,0 +1,129 @@ +--- +title: MAE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual + values. +--- + +# Metric Card for MAE + + +## Metric Description + +Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual numeric values: +![image](https://user-images.githubusercontent.com/14205986/165824243-e1078dfd-489d-456c-a0da-cbaa28726220.png) + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mae_metric = evaluate.load("mae") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mae_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MAE `float` value ranges from `0.0` to `+inf`, with the best value being 0.0. + +Output Example(s): +```python +{'mae': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mae': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mae_metric = evaluate.load("mae") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mae_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mae': 0.5} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mae_metric = evaluate.load("mae", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0, 2], [-1, 2], [8, -5]] +>>> results = mae_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mae': 0.75} +>>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mae': array([0.5, 1. ])} +``` + +## Limitations and Bias +One limitation of MAE is that the relative size of the error is not always obvious, meaning that it can be difficult to tell a big error from a smaller one -- metrics such as Mean Absolute Percentage Error (MAPE) have been proposed to calculate MAE in percentage terms. + +Also, since it calculates the mean, MAE may underestimate the impact of big, but infrequent, errors -- metrics such as the Root Mean Square Error (RMSE) compensate for this by taking the root of error values. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{willmott2005advantages, + title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, + author={Willmott, Cort J and Matsuura, Kenji}, + journal={Climate research}, + volume={30}, + number={1}, + pages={79--82}, + year={2005} +} +``` + +## Further References +- [Mean Absolute Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_error) diff --git a/ST/evaluate/metrics/mae/app.py b/ST/evaluate/metrics/mae/app.py new file mode 100644 index 0000000000000000000000000000000000000000..be2c05aa85e7eb06667955a25d79b4a2f0389419 --- /dev/null +++ b/ST/evaluate/metrics/mae/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mae") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mae/mae.py b/ST/evaluate/metrics/mae/mae.py new file mode 100644 index 0000000000000000000000000000000000000000..e973bc5ff60926eb28e7daebf20c00bc639a40db --- /dev/null +++ b/ST/evaluate/metrics/mae/mae.py @@ -0,0 +1,113 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MAE - Mean Absolute Error Metric""" + +import datasets +from sklearn.metrics import mean_absolute_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual +values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mae : mean absolute error. + If multioutput is "raw_values", then mean absolute error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MAE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mae_metric = evaluate.load("mae") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mae_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mae': 0.5} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mae_metric = evaluate.load("mae", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0, 2], [-1, 2], [8, -5]] + >>> results = mae_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mae': 0.75} + >>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'mae': array([0.5, 1. ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mae(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + mae_score = mean_absolute_error(references, predictions, sample_weight=sample_weight, multioutput=multioutput) + + return {"mae": mae_score} diff --git a/ST/evaluate/metrics/mae/requirements.txt b/ST/evaluate/metrics/mae/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/mae/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/mahalanobis/README.md b/ST/evaluate/metrics/mahalanobis/README.md new file mode 100644 index 0000000000000000000000000000000000000000..18a5ab4755a8af68bef1afcdee4f86ebabcf49b7 --- /dev/null +++ b/ST/evaluate/metrics/mahalanobis/README.md @@ -0,0 +1,92 @@ +--- +title: Mahalanobis Distance +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Compute the Mahalanobis Distance + + Mahalonobis distance is the distance between a point and a distribution. + And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance. + It was introduced by Prof. P. C. Mahalanobis in 1936 + and has been used in various statistical applications ever since + [source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/] +--- + +# Metric Card for Mahalanobis Distance + +## Metric Description +Mahalonobis distance is the distance between a point and a distribution (as opposed to the distance between two points), making it the multivariate equivalent of the Euclidean distance. + +It is often used in multivariate anomaly detection, classification on highly imbalanced datasets and one-class classification. + +## How to Use +At minimum, this metric requires two `list`s of datapoints: + +```python +>>> mahalanobis_metric = evaluate.load("mahalanobis") +>>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) +``` + +### Inputs +- `X` (`list`): data points to be compared with the `reference_distribution`. +- `reference_distribution` (`list`): data points from the reference distribution that we want to compare to. + +### Output Values +`mahalanobis` (`array`): the Mahalonobis distance for each data point in `X`. + +```python +>>> print(results) +{'mahalanobis': array([0.5])} +``` + +#### Values from Popular Papers +*N/A* + +### Example + +```python +>>> mahalanobis_metric = evaluate.load("mahalanobis") +>>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) +>>> print(results) +{'mahalanobis': array([0.5])} +``` + +## Limitations and Bias + +The Mahalanobis distance is only able to capture linear relationships between the variables, which means it cannot capture all types of outliers. Mahalanobis distance also fails to faithfully represent data that is highly skewed or multimodal. + +## Citation +```bibtex +@inproceedings{mahalanobis1936generalized, + title={On the generalized distance in statistics}, + author={Mahalanobis, Prasanta Chandra}, + year={1936}, + organization={National Institute of Science of India} +} +``` + +```bibtex +@article{de2000mahalanobis, + title={The Mahalanobis distance}, + author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, + journal={Chemometrics and intelligent laboratory systems}, + volume={50}, + number={1}, + pages={1--18}, + year={2000}, + publisher={Elsevier} +} +``` + +## Further References +-[Wikipedia -- Mahalanobis Distance](https://en.wikipedia.org/wiki/Mahalanobis_distance) + +-[Machine Learning Plus -- Mahalanobis Distance](https://www.machinelearningplus.com/statistics/mahalanobis-distance/) diff --git a/ST/evaluate/metrics/mahalanobis/app.py b/ST/evaluate/metrics/mahalanobis/app.py new file mode 100644 index 0000000000000000000000000000000000000000..cb6d338dc126a4131e80f466f605b9fd958bf884 --- /dev/null +++ b/ST/evaluate/metrics/mahalanobis/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mahalanobis") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mahalanobis/mahalanobis.py b/ST/evaluate/metrics/mahalanobis/mahalanobis.py new file mode 100644 index 0000000000000000000000000000000000000000..a2cad49964e11da10049ee82e8320d36fca01121 --- /dev/null +++ b/ST/evaluate/metrics/mahalanobis/mahalanobis.py @@ -0,0 +1,100 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mahalanobis metric.""" + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Compute the Mahalanobis Distance + +Mahalonobis distance is the distance between a point and a distribution. +And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance. +It was introduced by Prof. P. C. Mahalanobis in 1936 +and has been used in various statistical applications ever since +[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/] +""" + +_CITATION = """\ +@article{de2000mahalanobis, + title={The mahalanobis distance}, + author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, + journal={Chemometrics and intelligent laboratory systems}, + volume={50}, + number={1}, + pages={1--18}, + year={2000}, + publisher={Elsevier} +} +""" + +_KWARGS_DESCRIPTION = """ +Args: + X: List of datapoints to be compared with the `reference_distribution`. + reference_distribution: List of datapoints from the reference distribution we want to compare to. +Returns: + mahalanobis: The Mahalonobis distance for each datapoint in `X`. +Examples: + + >>> mahalanobis_metric = evaluate.load("mahalanobis") + >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) + >>> print(results) + {'mahalanobis': array([0.5])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mahalanobis(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "X": datasets.Sequence(datasets.Value("float", id="sequence"), id="X"), + } + ), + ) + + def _compute(self, X, reference_distribution): + + # convert to numpy arrays + X = np.array(X) + reference_distribution = np.array(reference_distribution) + + # Assert that arrays are 2D + if len(X.shape) != 2: + raise ValueError("Expected `X` to be a 2D vector") + if len(reference_distribution.shape) != 2: + raise ValueError("Expected `reference_distribution` to be a 2D vector") + if reference_distribution.shape[0] < 2: + raise ValueError( + "Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension" + ) + + # Get mahalanobis distance for each prediction + X_minus_mu = X - np.mean(reference_distribution) + cov = np.cov(reference_distribution.T) + try: + inv_covmat = np.linalg.inv(cov) + except np.linalg.LinAlgError: + inv_covmat = np.linalg.pinv(cov) + left_term = np.dot(X_minus_mu, inv_covmat) + mahal_dist = np.dot(left_term, X_minus_mu.T).diagonal() + + return {"mahalanobis": mahal_dist} diff --git a/ST/evaluate/metrics/mahalanobis/requirements.txt b/ST/evaluate/metrics/mahalanobis/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/mahalanobis/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/mape/README.md b/ST/evaluate/metrics/mape/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f430a881a23934612e8001c720bc82ac476766c6 --- /dev/null +++ b/ST/evaluate/metrics/mape/README.md @@ -0,0 +1,129 @@ +--- +title: MAPE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Percentage Error (MAPE) is the mean percentage error difference between the predicted and actual + values. +--- + +# Metric Card for MAPE + + +## Metric Description + +Mean Absolute Error (MAPE) is the mean of the percentage error of difference between the predicted $x_i$ and actual $y_i$ numeric values: +![image](https://user-images.githubusercontent.com/8100/200005316-c3975d32-8978-40f3-b541-c2ef57ec7c5b.png) + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mape_metric = evaluate.load("mape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mape_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MAPE `float` value is postive with the best value being 0.0. + +Output Example(s): +```python +{'mape': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mape': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mape_metric = evaluate.load("mape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mape': 0.3273...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mape_metric = evaluate.load("mape", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> results = mape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mape': 0.8874...} +>>> results = mape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mape': array([1.3749..., 0.4])} +``` + +## Limitations and Bias +One limitation of MAPE is that it cannot be used if the ground truth is zero or close to zero. This metric is also asymmetric in that it puts a heavier penalty on predictions less than the ground truth and a smaller penalty on predictions bigger than the ground truth and thus can lead to a bias of methods being select which under-predict if selected via this metric. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{DEMYTTENAERE201638, + title = {Mean Absolute Percentage Error for regression models}, + journal = {Neurocomputing}, + volume = {192}, + pages = {38--48}, + year = {2016}, + note = {Advances in artificial neural networks, machine learning and computational intelligence}, + issn = {0925-2312}, + doi = {https://doi.org/10.1016/j.neucom.2015.12.114}, + url = {https://www.sciencedirect.com/science/article/pii/S0925231216003325}, + author = {Arnaud {de Myttenaere} and Boris Golden and Bénédicte {Le Grand} and Fabrice Rossi}, +} +``` + +## Further References +- [Mean absolute percentage error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error) diff --git a/ST/evaluate/metrics/mape/app.py b/ST/evaluate/metrics/mape/app.py new file mode 100644 index 0000000000000000000000000000000000000000..8d8d20d70e85b33e0aacdf90117dac45054f6550 --- /dev/null +++ b/ST/evaluate/metrics/mape/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mape") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mape/mape.py b/ST/evaluate/metrics/mape/mape.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c52cb3cd6f32408c14c163f5a16b47c141fbd1 --- /dev/null +++ b/ST/evaluate/metrics/mape/mape.py @@ -0,0 +1,118 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MAPE - Mean Absolute Percentage Error Metric""" + +import datasets +from sklearn.metrics import mean_absolute_percentage_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Absolute Percentage Error (MAPE) is the mean percentage error difference between the predicted and actual +values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mape : mean absolute percentage error. + If multioutput is "raw_values", then mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MAPE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mape_metric = evaluate.load("mape") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mape': 0.3273809523809524} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mape_metric = evaluate.load("mape", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0.1, 2], [-1, 2], [8, -5]] + >>> results = mape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mape': 0.8874999875823658} + >>> results = mape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'mape': array([1.37499998, 0.4 ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mape(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_percentage_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + mape_score = mean_absolute_percentage_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + return {"mape": mape_score} diff --git a/ST/evaluate/metrics/mape/requirements.txt b/ST/evaluate/metrics/mape/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/ST/evaluate/metrics/mape/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/ST/evaluate/metrics/mase/README.md b/ST/evaluate/metrics/mase/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bded6399cb98326bc04e85797f45a327e21333ea --- /dev/null +++ b/ST/evaluate/metrics/mase/README.md @@ -0,0 +1,127 @@ +--- +title: MASE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast on the training set. +--- + +# Metric Card for MASE + +## Metric Description + +Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast. For prediction $x_i$ and corresponding ground truth $y_i$ as well as training data $z_t$ with seasonality $p$ the metric is given by: +![image](https://user-images.githubusercontent.com/8100/200009284-7ce4ccaa-373c-42f0-acbb-f81d52a97512.png) + + +This metric is: +* independent of the scale of the data; +* has predictable behavior when predicted/ground-truth data is near zero; +* symmetric; +* interpretable, as values greater than one indicate that in-sample one-step forecasts from the naïve method perform better than the forecast values under consideration. + + +## How to Use + +At minimum, this metric requires predictions, references and training data as inputs. + +```python +>>> mase_metric = evaluate.load("mase") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> training = [5, 0.5, 4, 6, 3, 5, 2] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. +- `training`: numeric array-like of shape (`n_train_samples,`) or (`n_train_samples`, `n_outputs`), representing the in sample training data. + +Optional arguments: +- `periodicity`: the seasonal periodicity of training data. The default is 1. +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MASE `float` value ranges from `0.0` to `1.0`, with the best value being 0.0. + +Output Example(s): +```python +{'mase': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mase': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mase_metric = evaluate.load("mase") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> training = [5, 0.5, 4, 6, 3, 5, 2] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +>>> print(results) +{'mase': 0.1833...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mase_metric = evaluate.load("mase", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> training = [[0.5, 1], [-1, 1], [7, -6]] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +>>> print(results) +{'mase': 0.1818...} +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput='raw_values') +>>> print(results) +{'mase': array([0.1052..., 0.2857...])} +``` + +## Limitations and Bias + + +## Citation(s) + +```bibtex +@article{HYNDMAN2006679, + title = {Another look at measures of forecast accuracy}, + journal = {International Journal of Forecasting}, + volume = {22}, + number = {4}, + pages = {679--688}, + year = {2006}, + issn = {0169-2070}, + doi = {https://doi.org/10.1016/j.ijforecast.2006.03.001}, + url = {https://www.sciencedirect.com/science/article/pii/S0169207006000239}, + author = {Rob J. Hyndman and Anne B. Koehler}, +} +``` + +## Further References +- [Mean absolute scaled error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_scaled_errorr) diff --git a/ST/evaluate/metrics/mase/app.py b/ST/evaluate/metrics/mase/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ac47c0d679868dd50c8b5476a1e53d721c429b90 --- /dev/null +++ b/ST/evaluate/metrics/mase/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mase") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mase/mase.py b/ST/evaluate/metrics/mase/mase.py new file mode 100644 index 0000000000000000000000000000000000000000..633d2e0140474066e50609bae578387d4998369a --- /dev/null +++ b/ST/evaluate/metrics/mase/mase.py @@ -0,0 +1,140 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MASE - Mean Absolute Scaled Error Metric""" + +import datasets +import numpy as np +from sklearn.metrics import mean_absolute_error + +import evaluate + + +_CITATION = """\ +@article{HYNDMAN2006679, + title = {Another look at measures of forecast accuracy}, + journal = {International Journal of Forecasting}, + volume = {22}, + number = {4}, + pages = {679--688}, + year = {2006}, + issn = {0169-2070}, + doi = {https://doi.org/10.1016/j.ijforecast.2006.03.001}, + url = {https://www.sciencedirect.com/science/article/pii/S0169207006000239}, + author = {Rob J. Hyndman and Anne B. Koehler}, +} +""" + +_DESCRIPTION = """\ +Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + training: array-like of shape (n_train_samples,) or (n_train_samples, n_outputs) + In sample training data for naive forecast. + periodicity: int, default=1 + Seasonal periodicity of training data. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mase : mean absolute scaled error. + If multioutput is "raw_values", then mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MASE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mase_metric = evaluate.load("mase") + >>> predictions = [2.5, 0.0, 2, 8, 1.25] + >>> references = [3, -0.5, 2, 7, 2] + >>> training = [5, 0.5, 4, 6, 3, 5, 2] + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training) + >>> print(results) + {'mase': 0.18333333333333335} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mase_metric = evaluate.load("mase", "multilist") + >>> predictions = [[0, 2], [-1, 2], [8, -5]] + >>> references = [[0.5, 1], [-1, 1], [7, -6]] + >>> training = [[0.5, 1], [-1, 1], [7, -6]] + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training) + >>> print(results) + {'mase': 0.18181818181818182} + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput='raw_values') + >>> print(results) + {'mase': array([0.10526316, 0.28571429])} + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput=[0.3, 0.7]) + >>> print(results) + {'mase': 0.21935483870967742} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mase(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=["https://otexts.com/fpp3/accuracy.html#scaled-errors"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute( + self, + predictions, + references, + training, + periodicity=1, + sample_weight=None, + multioutput="uniform_average", + ): + + y_pred_naive = training[:-periodicity] + mae_naive = mean_absolute_error(training[periodicity:], y_pred_naive, multioutput=multioutput) + + mae_score = mean_absolute_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + epsilon = np.finfo(np.float64).eps + mase_score = mae_score / np.maximum(mae_naive, epsilon) + + return {"mase": mase_score} diff --git a/ST/evaluate/metrics/mase/requirements.txt b/ST/evaluate/metrics/mase/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/ST/evaluate/metrics/mase/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/ST/evaluate/metrics/matthews_correlation/README.md b/ST/evaluate/metrics/matthews_correlation/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1876358cb3d65e64fab0e3ef5c79531df3f285e6 --- /dev/null +++ b/ST/evaluate/metrics/matthews_correlation/README.md @@ -0,0 +1,117 @@ +--- +title: Matthews Correlation Coefficient +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Compute the Matthews correlation coefficient (MCC) + + The Matthews correlation coefficient is used in machine learning as a + measure of the quality of binary and multiclass classifications. It takes + into account true and false positives and negatives and is generally + regarded as a balanced measure which can be used even if the classes are of + very different sizes. The MCC is in essence a correlation coefficient value + between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 + an average random prediction and -1 an inverse prediction. The statistic + is also known as the phi coefficient. [source: Wikipedia] +--- + +# Metric Card for Matthews Correlation Coefficient + +## Metric Description +The Matthews correlation coefficient is used in machine learning as a +measure of the quality of binary and multiclass classifications. It takes +into account true and false positives and negatives and is generally +regarded as a balanced measure which can be used even if the classes are of +very different sizes. The MCC is in essence a correlation coefficient value +between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 +an average random prediction and -1 an inverse prediction. The statistic +is also known as the phi coefficient. [source: Wikipedia] + +## How to Use +At minimum, this metric requires a list of predictions and a list of references: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'matthews_correlation': 1.0} +``` + +### Inputs +- **`predictions`** (`list` of `int`s): Predicted class labels. +- **`references`** (`list` of `int`s): Ground truth labels. +- **`sample_weight`** (`list` of `int`s, `float`s, or `bool`s): Sample weights. Defaults to `None`. +- **`average`**(`None` or `macro`): For the multilabel case, whether to return one correlation coefficient per feature (`average=None`), or the average of them (`average='macro'`). Defaults to `None`. + +### Output Values +- **`matthews_correlation`** (`float` or `list` of `float`s): Matthews correlation coefficient, or list of them in the multilabel case without averaging. + +The metric output takes the following form: +```python +{'matthews_correlation': 0.54} +``` + +This metric can be any value from -1 to +1, inclusive. + +#### Values from Popular Papers + + +### Examples +A basic example with only predictions and references as inputs: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3]) +>>> print(results) +{'matthews_correlation': 0.5384615384615384} +``` + +The same example as above, but also including sample weights: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3], +... sample_weight=[0.5, 3, 1, 1, 1, 2]) +>>> print(results) +{'matthews_correlation': 0.09782608695652174} +``` + +The same example as above, with sample weights that cause a negative correlation: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3], +... sample_weight=[0.5, 1, 0, 0, 0, 1]) +>>> print(results) +{'matthews_correlation': -0.25} +``` + +## Limitations and Bias +*Note any limitations or biases that the metric has.* + + +## Citation +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +## Further References + +- This Hugging Face implementation uses [this scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html) \ No newline at end of file diff --git a/ST/evaluate/metrics/matthews_correlation/app.py b/ST/evaluate/metrics/matthews_correlation/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d1018fbc378cd089a599d63f74e4b05bc10147 --- /dev/null +++ b/ST/evaluate/metrics/matthews_correlation/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("matthews_correlation") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/matthews_correlation/matthews_correlation.py b/ST/evaluate/metrics/matthews_correlation/matthews_correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..58a477a928113558a68c019540daadd206d56458 --- /dev/null +++ b/ST/evaluate/metrics/matthews_correlation/matthews_correlation.py @@ -0,0 +1,140 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Matthews Correlation metric.""" + +import datasets +import numpy as np +from sklearn.metrics import matthews_corrcoef + +import evaluate + + +_DESCRIPTION = """ +Compute the Matthews correlation coefficient (MCC) + +The Matthews correlation coefficient is used in machine learning as a +measure of the quality of binary and multiclass classifications. It takes +into account true and false positives and negatives and is generally +regarded as a balanced measure which can be used even if the classes are of +very different sizes. The MCC is in essence a correlation coefficient value +between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 +an average random prediction and -1 an inverse prediction. The statistic +is also known as the phi coefficient. [source: Wikipedia] +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (list of int): Predicted labels, as returned by a model. + references (list of int): Ground truth labels. + average (`string`): This parameter is used for multilabel configs. Defaults to `None`. + - None (default): Returns an array of Matthews correlation coefficients, one for each feature + - 'macro': Calculate metrics for each feature, and find their unweighted mean. + sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`. +Returns: + matthews_correlation (dict containing float): Matthews correlation. +Examples: + Example 1, a basic example with only predictions and references as inputs: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3]) + >>> print(round(results['matthews_correlation'], 2)) + 0.54 + + Example 2, the same example as above, but also including sample weights: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3], + ... sample_weight=[0.5, 3, 1, 1, 1, 2]) + >>> print(round(results['matthews_correlation'], 2)) + 0.1 + + Example 3, the same example as above, but with sample weights that cause a negative correlation: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3], + ... sample_weight=[0.5, 1, 0, 0, 0, 1]) + >>> print(round(results['matthews_correlation'], 2)) + -0.25 + + Example 4, Multi-label without averaging: + >>> matthews_metric = evaluate.load("matthews_correlation", config_name="multilabel") + >>> results = matthews_metric.compute(references=[[0,1], [1,0], [1,1]], + ... predictions=[[0,1], [1,1], [0,1]]) + >>> print(results['matthews_correlation']) + [0.5, 0.0] + + Example 5, Multi-label with averaging: + >>> matthews_metric = evaluate.load("matthews_correlation", config_name="multilabel") + >>> results = matthews_metric.compute(references=[[0,1], [1,0], [1,1]], + ... predictions=[[0,1], [1,1], [0,1]], + ... average='macro') + >>> print(round(results['matthews_correlation'], 2)) + 0.25 +""" + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class MatthewsCorrelation(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html" + ], + ) + + def _compute(self, predictions, references, sample_weight=None, average=None): + if self.config_name == "multilabel": + references = np.array(references) + predictions = np.array(predictions) + if not (references.ndim == 2 and predictions.ndim == 2): + raise ValueError("For multi-label inputs, both references and predictions should be 2-dimensional") + matthews_corr = [ + matthews_corrcoef(predictions[:, i], references[:, i], sample_weight=sample_weight) + for i in range(references.shape[1]) + ] + if average == "macro": + matthews_corr = np.mean(matthews_corr) + elif average is not None: + raise ValueError("Invalid `average`: expected `macro`, or None ") + else: + matthews_corr = float(matthews_corrcoef(references, predictions, sample_weight=sample_weight)) + return {"matthews_correlation": matthews_corr} diff --git a/ST/evaluate/metrics/matthews_correlation/requirements.txt b/ST/evaluate/metrics/matthews_correlation/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/matthews_correlation/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/mauve/README.md b/ST/evaluate/metrics/mauve/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ba39d8fad7a96dc09227b1bae7ff849cd05f165 --- /dev/null +++ b/ST/evaluate/metrics/mauve/README.md @@ -0,0 +1,144 @@ +--- +title: MAUVE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + MAUVE is a measure of the statistical gap between two text distributions, e.g., how far the text written by a model is the distribution of human text, using samples from both distributions. + + MAUVE is obtained by computing Kullback–Leibler (KL) divergences between the two distributions in a quantized embedding space of a large language model. It can quantify differences in the quality of generated text based on the size of the model, the decoding algorithm, and the length of the generated text. MAUVE was found to correlate the strongest with human evaluations over baseline metrics for open-ended text generation. + +--- + +# Metric Card for MAUVE + +## Metric description + +MAUVE is a measure of the gap between neural text and human text. It is computed using the [Kullback–Leibler (KL) divergences](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) between the two distributions of text in a quantized embedding space of a large language model. MAUVE can identify differences in quality arising from model sizes and decoding algorithms. + +This metric is a wrapper around the [official implementation](https://github.com/krishnap25/mauve) of MAUVE. + +For more details, consult the [MAUVE paper](https://arxiv.org/abs/2102.01454). + +## How to use + +The metric takes two lists of strings of tokens separated by spaces: one representing `predictions` (i.e. the text generated by the model) and the second representing `references` (a reference text for each prediction): + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello world", "goodnight moon"] +mauve_results = mauve.compute(predictions=predictions, references=references) +``` + +It also has several optional arguments: + +`num_buckets`: the size of the histogram to quantize P and Q. Options: `auto` (default) or an integer. + +`pca_max_data`: the number of data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. The default is `-1`. + +`kmeans_explained_var`: the amount of variance of the data to keep in dimensionality reduction by PCA. The default is `0.9`. + +`kmeans_num_redo`: number of times to redo k-means clustering (the best objective is kept). The default is `5`. + +`kmeans_max_iter`: maximum number of k-means iterations. The default is `500`. + +`featurize_model_name`: name of the model from which features are obtained, from one of the following: `gpt2`, `gpt2-medium`, `gpt2-large`, `gpt2-xl`. The default is `gpt2-large`. + +`device_id`: Device for featurization. Supply a GPU id (e.g. `0` or `3`) to use GPU. If no GPU with this id is found, the metric will use CPU. + +`max_text_length`: maximum number of tokens to consider. The default is `1024`. + +`divergence_curve_discretization_size` Number of points to consider on the divergence curve. The default is `25`. + +`mauve_scaling_factor`: Hyperparameter for scaling. The default is `5`. + +`verbose`: If `True` (default), running the metric will print running time updates. + +`seed`: random seed to initialize k-means cluster assignments, randomly assigned by default. + + + +## Output values + +This metric outputs a dictionary with 5 key-value pairs: + +`mauve`: MAUVE score, which ranges between 0 and 1. **Larger** values indicate that P and Q are closer. + +`frontier_integral`: Frontier Integral, which ranges between 0 and 1. **Smaller** values indicate that P and Q are closer. + +`divergence_curve`: a numpy.ndarray of shape (m, 2); plot it with `matplotlib` to view the divergence curve. + +`p_hist`: a discrete distribution, which is a quantized version of the text distribution `p_text`. + +`q_hist`: same as above, but with `q_text`. + + +### Values from popular papers + +The [original MAUVE paper](https://arxiv.org/abs/2102.01454) reported values ranging from 0.88 to 0.94 for open-ended text generation using a text completion task in the web text domain. The authors found that bigger models resulted in higher MAUVE scores and that MAUVE is correlated with human judgments. + + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello world", "goodnight moon"] +mauve_results = mauve.compute(predictions=predictions, references=references) +print(mauve_results.mauve) +1.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello there", "general kenobi"] +mauve_results = mauve.compute(predictions=predictions, references=references) +print(mauve_results.mauve) +0.27811372536724027 +``` + +## Limitations and bias + +The [original MAUVE paper](https://arxiv.org/abs/2102.01454) did not analyze the inductive biases present in different embedding models, but related work has shown different kinds of biases exist in many popular generative language models including GPT-2 (see [Kirk et al., 2021](https://arxiv.org/pdf/2102.04130.pdf), [Abid et al., 2021](https://arxiv.org/abs/2101.05783)). The extent to which these biases can impact the MAUVE score has not been quantified. + +Also, calculating the MAUVE metric involves downloading the model from which features are obtained -- the default model, `gpt2-large`, takes over 3GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance, `gpt` is 523MB. + +It is a good idea to use at least 1000 samples for each distribution to compute MAUVE (the original paper uses 5000). + +MAUVE is unable to identify very small differences between different settings of generation (e.g., between top-p sampling with p=0.95 versus 0.96). It is important, therefore, to account for the randomness inside the generation (e.g., due to sampling) and within the MAUVE estimation procedure (see the `seed` parameter above). Concretely, it is a good idea to obtain generations using multiple random seeds and/or to use rerun MAUVE with multiple values of the parameter `seed`. + +For MAUVE to be large, the model distribution must be close to the human text distribution as seen by the embeddings. It is possible to have high-quality model text that still has a small MAUVE score (i.e., large gap) if it contains text about different topics/subjects, or uses a different writing style or vocabulary, or contains texts of a different length distribution. MAUVE summarizes the statistical gap (as measured by the large language model embeddings) --- this includes all these factors in addition to the quality-related aspects such as grammaticality. + +See the [official implementation](https://github.com/krishnap25/mauve#best-practices-for-mauve) for more details about best practices. + + +## Citation + +```bibtex +@inproceedings{pillutla-etal:mauve:neurips2021, + title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}, + author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, + booktitle = {NeurIPS}, + year = {2021} +} +``` + +## Further References +- [Official MAUVE implementation](https://github.com/krishnap25/mauve) +- [Hugging Face Tasks - Text Generation](https://huggingface.co/tasks/text-generation) diff --git a/ST/evaluate/metrics/mauve/app.py b/ST/evaluate/metrics/mauve/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d356aef0ad1640efd4963d8bd33588b1442a40c0 --- /dev/null +++ b/ST/evaluate/metrics/mauve/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("mauve") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mauve/mauve.py b/ST/evaluate/metrics/mauve/mauve.py new file mode 100644 index 0000000000000000000000000000000000000000..9969135a7e3f058aa8b77e52927b905f6b0551ec --- /dev/null +++ b/ST/evaluate/metrics/mauve/mauve.py @@ -0,0 +1,150 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MAUVE metric from https://github.com/krishnap25/mauve. """ + +import datasets +import faiss # Here to have a nice missing dependency error message early on +import numpy # Here to have a nice missing dependency error message early on +import requests # Here to have a nice missing dependency error message early on +import sklearn # Here to have a nice missing dependency error message early on +import tqdm # Here to have a nice missing dependency error message early on +from mauve import compute_mauve # From: mauve-text + +import evaluate + + +_CITATION = """\ +@inproceedings{pillutla-etal:mauve:neurips2021, + title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}, + author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, + booktitle = {NeurIPS}, + year = {2021} +} + +""" + +_DESCRIPTION = """\ +MAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure. + +MAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences. + +For details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021). + +This metrics is a wrapper around the official implementation of MAUVE: +https://github.com/krishnap25/mauve +""" + +_KWARGS_DESCRIPTION = """ +Calculates MAUVE scores between two lists of generated text and reference text. +Args: + predictions: list of generated text to score. Each predictions + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. +Optional Args: + num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer + pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1 + kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9 + kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5 + kmeans_max_iter: maximum number of k-means iterations. Default 500 + featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl']. + device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU + max_text_length: maximum number of tokens to consider. Default 1024 + divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25 + mauve_scaling_factor: "c" from the paper. Default 5. + verbose: If True (default), print running time updates + seed: random seed to initialize k-means cluster assignments. +Returns: + mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer, + frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer, + divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve, + p_hist: a discrete distribution, which is a quantized version of the text distribution p_text, + q_hist: same as above, but with q_text. +Examples: + + >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest + >>> import evaluate + >>> mauve = evaluate.load('mauve') + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP + >>> print(out.mauve) # doctest: +SKIP + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mauve(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/krishnap25/mauve", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/krishnap25/mauve"], + reference_urls=[ + "https://arxiv.org/abs/2102.01454", + "https://github.com/krishnap25/mauve", + ], + ) + + def _compute( + self, + predictions, + references, + p_features=None, + q_features=None, + p_tokens=None, + q_tokens=None, + num_buckets="auto", + pca_max_data=-1, + kmeans_explained_var=0.9, + kmeans_num_redo=5, + kmeans_max_iter=500, + featurize_model_name="gpt2-large", + device_id=-1, + max_text_length=1024, + divergence_curve_discretization_size=25, + mauve_scaling_factor=5, + verbose=True, + seed=25, + ): + out = compute_mauve( + p_text=predictions, + q_text=references, + p_features=p_features, + q_features=q_features, + p_tokens=p_tokens, + q_tokens=q_tokens, + num_buckets=num_buckets, + pca_max_data=pca_max_data, + kmeans_explained_var=kmeans_explained_var, + kmeans_num_redo=kmeans_num_redo, + kmeans_max_iter=kmeans_max_iter, + featurize_model_name=featurize_model_name, + device_id=device_id, + max_text_length=max_text_length, + divergence_curve_discretization_size=divergence_curve_discretization_size, + mauve_scaling_factor=mauve_scaling_factor, + verbose=verbose, + seed=seed, + ) + return out diff --git a/ST/evaluate/metrics/mauve/requirements.txt b/ST/evaluate/metrics/mauve/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..dbab93c719102f0a0a18219eb2a555396dda6967 --- /dev/null +++ b/ST/evaluate/metrics/mauve/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +faiss-cpu +scikit-learn +mauve-text \ No newline at end of file diff --git a/ST/evaluate/metrics/mean_iou/README.md b/ST/evaluate/metrics/mean_iou/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d92d8019570cfe334310a71ff4df67bbcc5b3da7 --- /dev/null +++ b/ST/evaluate/metrics/mean_iou/README.md @@ -0,0 +1,108 @@ +--- +title: Mean IoU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union + between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation, + the mean IoU of the image is calculated by taking the IoU of each class and averaging them. +--- + +# Metric Card for Mean IoU + + +## Metric Description + +IoU (Intersection over Union) is the area of overlap between the predicted segmentation and the ground truth divided by the area of union between the predicted segmentation and the ground truth. + +For binary (two classes) or multi-class segmentation, the *mean IoU* of the image is calculated by taking the IoU of each class and averaging them. + +## How to Use + +The Mean IoU metric takes two numeric arrays as input corresponding to the predicted and ground truth segmentations: +```python +>>> import numpy as np +>>> mean_iou = evaluate.load("mean_iou") +>>> predicted = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) +>>> ground_truth = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) +>>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255) +``` + +### Inputs +**Mandatory inputs** +- `predictions` (`List[ndarray]`): List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. +- `references` (`List[ndarray]`): List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. +- `num_labels` (`int`): Number of classes (categories). +- `ignore_index` (`int`): Index that will be ignored during evaluation. + +**Optional inputs** +- `nan_to_num` (`int`): If specified, NaN values will be replaced by the number defined by the user. +- `label_map` (`dict`): If specified, dictionary mapping old label indices to new label indices. +- `reduce_labels` (`bool`): Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. The default value is `False`. + +### Output Values +The metric returns a dictionary with the following elements: +- `mean_iou` (`float`): Mean Intersection-over-Union (IoU averaged over all categories). +- `mean_accuracy` (`float`): Mean accuracy (averaged over all categories). +- `overall_accuracy` (`float`): Overall accuracy on all images. +- `per_category_accuracy` (`ndarray` of shape `(num_labels,)`): Per category accuracy. +- `per_category_iou` (`ndarray` of shape `(num_labels,)`): Per category IoU. + +The values of all of the scores reported range from from `0.0` (minimum) and `1.0` (maximum). + +Output Example: +```python +{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +``` + +#### Values from Popular Papers + +The [leaderboard for the CityScapes dataset](https://paperswithcode.com/sota/semantic-segmentation-on-cityscapes) reports a Mean IOU ranging from 64 to 84; that of [ADE20k](https://paperswithcode.com/sota/semantic-segmentation-on-ade20k) ranges from 30 to a peak of 59.9, indicating that the dataset is more difficult for current approaches (as of 2022). + + +### Examples + +```python +>>> import numpy as np +>>> mean_iou = evaluate.load("mean_iou") +>>> # suppose one has 3 different segmentation maps predicted +>>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) +>>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) +>>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) +>>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) +>>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) +>>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) +>>> predictions = [predicted_1, predicted_2, predicted_3] +>>> references = [actual_1, actual_2, actual_3] +>>> results = mean_iou.compute(predictions=predictions, references=references, num_labels=10, ignore_index=255, reduce_labels=False) +>>> print(results) # doctest: +NORMALIZE_WHITESPACE +{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +``` + + +## Limitations and Bias +Mean IOU is an average metric, so it will not show you where model predictions differ from the ground truth (i.e. if there are particular regions or classes that the model does poorly on). Further error analysis is needed to gather actional insights that can be used to inform model improvements. + +## Citation(s) +```bibtex +@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, +author = {{MMSegmentation Contributors}}, +license = {Apache-2.0}, +month = {7}, +title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, +url = {https://github.com/open-mmlab/mmsegmentation}, +year = {2020} +}" +``` + + +## Further References +- [Wikipedia article - Jaccard Index](https://en.wikipedia.org/wiki/Jaccard_index) diff --git a/ST/evaluate/metrics/mean_iou/app.py b/ST/evaluate/metrics/mean_iou/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5423d3579f8135484afdf5e24279b2f93df0e8d0 --- /dev/null +++ b/ST/evaluate/metrics/mean_iou/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mean_iou") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mean_iou/mean_iou.py b/ST/evaluate/metrics/mean_iou/mean_iou.py new file mode 100644 index 0000000000000000000000000000000000000000..421a261f41d4ddb5782ed005e0405560494cc378 --- /dev/null +++ b/ST/evaluate/metrics/mean_iou/mean_iou.py @@ -0,0 +1,314 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mean IoU (Intersection-over-Union) metric.""" + +from typing import Dict, Optional + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union +between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation, +the mean IoU of the image is calculated by taking the IoU of each class and averaging them. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`List[ndarray]`): + List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + references (`List[ndarray]`): + List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + num_labels (`int`): + Number of classes (categories). + ignore_index (`int`): + Index that will be ignored during evaluation. + nan_to_num (`int`, *optional*): + If specified, NaN values will be replaced by the number defined by the user. + label_map (`dict`, *optional*): + If specified, dictionary mapping old label indices to new label indices. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + +Returns: + `Dict[str, float | ndarray]` comprising various elements: + - *mean_iou* (`float`): + Mean Intersection-over-Union (IoU averaged over all categories). + - *mean_accuracy* (`float`): + Mean accuracy (averaged over all categories). + - *overall_accuracy* (`float`): + Overall accuracy on all images. + - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): + Per category accuracy. + - *per_category_iou* (`ndarray` of shape `(num_labels,)`): + Per category IoU. + +Examples: + + >>> import numpy as np + + >>> mean_iou = evaluate.load("mean_iou") + + >>> # suppose one has 3 different segmentation maps predicted + >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) + >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) + + >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) + >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) + + >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) + >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) + + >>> predicted = [predicted_1, predicted_2, predicted_3] + >>> ground_truth = [actual_1, actual_2, actual_3] + + >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False) + >>> print(results) # doctest: +NORMALIZE_WHITESPACE + {'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +""" + +_CITATION = """\ +@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, +author = {{MMSegmentation Contributors}}, +license = {Apache-2.0}, +month = {7}, +title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, +url = {https://github.com/open-mmlab/mmsegmentation}, +year = {2020} +}""" + + +def intersect_and_union( + pred_label, + label, + num_labels, + ignore_index: bool, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate intersection and Union. + + Args: + pred_label (`ndarray`): + Prediction segmentation map of shape (height, width). + label (`ndarray`): + Ground truth segmentation map of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + area_intersect (`ndarray`): + The intersection of prediction and ground truth histogram on all classes. + area_union (`ndarray`): + The union of prediction and ground truth histogram on all classes. + area_pred_label (`ndarray`): + The prediction histogram on all classes. + area_label (`ndarray`): + The ground truth histogram on all classes. + """ + if label_map is not None: + for old_id, new_id in label_map.items(): + label[label == old_id] = new_id + + # turn into Numpy arrays + pred_label = np.array(pred_label) + label = np.array(label) + + if reduce_labels: + label[label == 0] = 255 + label = label - 1 + label[label == 254] = 255 + + mask = label != ignore_index + mask = np.not_equal(label, ignore_index) + pred_label = pred_label[mask] + label = np.array(label)[mask] + + intersect = pred_label[pred_label == label] + + area_intersect = np.histogram(intersect, bins=num_labels, range=(0, num_labels - 1))[0] + area_pred_label = np.histogram(pred_label, bins=num_labels, range=(0, num_labels - 1))[0] + area_label = np.histogram(label, bins=num_labels, range=(0, num_labels - 1))[0] + + area_union = area_pred_label + area_label - area_intersect + + return area_intersect, area_union, area_pred_label, area_label + + +def total_intersect_and_union( + results, + gt_seg_maps, + num_labels, + ignore_index: bool, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate Total Intersection and Union, by calculating `intersect_and_union` for each (predicted, ground truth) pair. + + Args: + results (`ndarray`): + List of prediction segmentation maps, each of shape (height, width). + gt_seg_maps (`ndarray`): + List of ground truth segmentation maps, each of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + total_area_intersect (`ndarray`): + The intersection of prediction and ground truth histogram on all classes. + total_area_union (`ndarray`): + The union of prediction and ground truth histogram on all classes. + total_area_pred_label (`ndarray`): + The prediction histogram on all classes. + total_area_label (`ndarray`): + The ground truth histogram on all classes. + """ + total_area_intersect = np.zeros((num_labels,), dtype=np.float64) + total_area_union = np.zeros((num_labels,), dtype=np.float64) + total_area_pred_label = np.zeros((num_labels,), dtype=np.float64) + total_area_label = np.zeros((num_labels,), dtype=np.float64) + for result, gt_seg_map in zip(results, gt_seg_maps): + area_intersect, area_union, area_pred_label, area_label = intersect_and_union( + result, gt_seg_map, num_labels, ignore_index, label_map, reduce_labels + ) + total_area_intersect += area_intersect + total_area_union += area_union + total_area_pred_label += area_pred_label + total_area_label += area_label + return total_area_intersect, total_area_union, total_area_pred_label, total_area_label + + +def mean_iou( + results, + gt_seg_maps, + num_labels, + ignore_index: bool, + nan_to_num: Optional[int] = None, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate Mean Intersection and Union (mIoU). + + Args: + results (`ndarray`): + List of prediction segmentation maps, each of shape (height, width). + gt_seg_maps (`ndarray`): + List of ground truth segmentation maps, each of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + nan_to_num (`int`, *optional*): + If specified, NaN values will be replaced by the number defined by the user. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + `Dict[str, float | ndarray]` comprising various elements: + - *mean_iou* (`float`): + Mean Intersection-over-Union (IoU averaged over all categories). + - *mean_accuracy* (`float`): + Mean accuracy (averaged over all categories). + - *overall_accuracy* (`float`): + Overall accuracy on all images. + - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): + Per category accuracy. + - *per_category_iou* (`ndarray` of shape `(num_labels,)`): + Per category IoU. + """ + total_area_intersect, total_area_union, total_area_pred_label, total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_labels, ignore_index, label_map, reduce_labels + ) + + # compute metrics + metrics = dict() + + all_acc = total_area_intersect.sum() / total_area_label.sum() + iou = total_area_intersect / total_area_union + acc = total_area_intersect / total_area_label + + metrics["mean_iou"] = np.nanmean(iou) + metrics["mean_accuracy"] = np.nanmean(acc) + metrics["overall_accuracy"] = all_acc + metrics["per_category_iou"] = iou + metrics["per_category_accuracy"] = acc + + if nan_to_num is not None: + metrics = dict( + {metric: np.nan_to_num(metric_value, nan=nan_to_num) for metric, metric_value in metrics.items()} + ) + + return metrics + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class MeanIoU(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + # 1st Seq - height dim, 2nd - width dim + { + "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + } + ), + reference_urls=[ + "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py" + ], + ) + + def _compute( + self, + predictions, + references, + num_labels: int, + ignore_index: bool, + nan_to_num: Optional[int] = None, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, + ): + iou_result = mean_iou( + results=predictions, + gt_seg_maps=references, + num_labels=num_labels, + ignore_index=ignore_index, + nan_to_num=nan_to_num, + label_map=label_map, + reduce_labels=reduce_labels, + ) + return iou_result diff --git a/ST/evaluate/metrics/mean_iou/requirements.txt b/ST/evaluate/metrics/mean_iou/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/mean_iou/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/meteor/README.md b/ST/evaluate/metrics/meteor/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0b234f70f1816a9084d4f8898ac7db20eead0a90 --- /dev/null +++ b/ST/evaluate/metrics/meteor/README.md @@ -0,0 +1,139 @@ +--- +title: METEOR +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + METEOR, an automatic metric for machine translation evaluation + that is based on a generalized concept of unigram matching between the + machine-produced translation and human-produced reference translations. + Unigrams can be matched based on their surface forms, stemmed forms, + and meanings; furthermore, METEOR can be easily extended to include more + advanced matching strategies. Once all generalized unigram matches + between the two strings have been found, METEOR computes a score for + this matching using a combination of unigram-precision, unigram-recall, and + a measure of fragmentation that is designed to directly capture how + well-ordered the matched words in the machine translation are in relation + to the reference. + + METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic + data and 0.331 on the Chinese data. This is shown to be an improvement on + using simply unigram-precision, unigram-recall and their harmonic F1 + combination. +--- + +# Metric Card for METEOR + +## Metric description + +METEOR (Metric for Evaluation of Translation with Explicit ORdering) is a machine translation evaluation metric, which is calculated based on the harmonic mean of precision and recall, with recall weighted more than precision. + +METEOR is based on a generalized concept of unigram matching between the machine-produced translation and human-produced reference translations. Unigrams can be matched based on their surface forms, stemmed forms, and meanings. Once all generalized unigram matches between the two strings have been found, METEOR computes a score for this matching using a combination of unigram-precision, unigram-recall, and a measure of fragmentation that is designed to directly capture how well-ordered the matched words in the machine translation are in relation to the reference. + + +## How to use + +METEOR has two mandatory arguments: + +`predictions`: a `list` of predictions to score. Each prediction should be a string with tokens separated by spaces. + +`references`: a `list` of references (in the case of one `reference` per `prediction`), or a `list` of `lists` of references (in the case of multiple `references` per `prediction`. Each reference should be a string with tokens separated by spaces. + +It also has several optional parameters: + +`alpha`: Parameter for controlling relative weights of precision and recall. The default value is `0.9`. + +`beta`: Parameter for controlling shape of penalty as a function of fragmentation. The default value is `3`. + +`gamma`: The relative weight assigned to fragmentation penalty. The default is `0.5`. + +Refer to the [METEOR paper](https://aclanthology.org/W05-0909.pdf) for more information about parameter values and ranges. + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] +>>> results = meteor.compute(predictions=predictions, references=references) +``` + +## Output values + +The metric outputs a dictionary containing the METEOR score. Its values range from 0 to 1, e.g.: +``` +{'meteor': 0.9999142661179699} +``` + + +### Values from popular papers +The [METEOR paper](https://aclanthology.org/W05-0909.pdf) does not report METEOR score values for different models, but it does report that METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic data and 0.331 on the Chinese data. + + +## Examples + +One `reference` per `prediction`: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> reference = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> results = meteor.compute(predictions=predictions, references=reference) +>>> print(round(results['meteor'], 2)) +1.0 +``` + +Multiple `references` per `prediction`: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = [['It is a guide to action that ensures that the military will forever heed Party commands', 'It is the guiding principle which guarantees the military forces always being under the command of the Party', 'It is the practical guide for the army always to heed the directions of the party']] +>>> results = meteor.compute(predictions=predictions, references=references) +>>> print(round(results['meteor'], 2)) +1.0 +``` + +Multiple `references` per `prediction`, partial match: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = [['It is a guide to action that ensures that the military will forever heed Party commands', 'It is the guiding principle which guarantees the military forces always being under the command of the Party', 'It is the practical guide for the army always to heed the directions of the party']] +>>> results = meteor.compute(predictions=predictions, references=references) +>>> print(round(results['meteor'], 2)) +0.69 +``` + +## Limitations and bias + +While the correlation between METEOR and human judgments was measured for Chinese and Arabic and found to be significant, further experimentation is needed to check its correlation for other languages. + +Furthermore, while the alignment and matching done in METEOR is based on unigrams, using multiple word entities (e.g. bigrams) could contribute to improving its accuracy -- this has been proposed in [more recent publications](https://www.cs.cmu.edu/~alavie/METEOR/pdf/meteor-naacl-2010.pdf) on the subject. + + +## Citation + +```bibtex +@inproceedings{banarjee2005, + title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, + author = {Banerjee, Satanjeev and Lavie, Alon}, + booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, + month = jun, + year = {2005}, + address = {Ann Arbor, Michigan}, + publisher = {Association for Computational Linguistics}, + url = {https://www.aclweb.org/anthology/W05-0909}, + pages = {65--72}, +} +``` + +## Further References +- [METEOR -- Wikipedia](https://en.wikipedia.org/wiki/METEOR) +- [METEOR score -- NLTK](https://www.nltk.org/_modules/nltk/translate/meteor_score.html) + diff --git a/ST/evaluate/metrics/meteor/app.py b/ST/evaluate/metrics/meteor/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3bcb6bf27d14ef87a84ffbdfda0721a13c811481 --- /dev/null +++ b/ST/evaluate/metrics/meteor/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("meteor") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/meteor/meteor.py b/ST/evaluate/metrics/meteor/meteor.py new file mode 100644 index 0000000000000000000000000000000000000000..7a6ec5cd4220730266f1c97d156f99fac5e16a88 --- /dev/null +++ b/ST/evaluate/metrics/meteor/meteor.py @@ -0,0 +1,162 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" METEOR metric. """ + +import datasets +import numpy as np +from datasets.config import importlib_metadata, version +from nltk.translate import meteor_score + +import evaluate + + +NLTK_VERSION = version.parse(importlib_metadata.version("nltk")) +if NLTK_VERSION >= version.Version("3.6.4"): + from nltk import word_tokenize + + +_CITATION = """\ +@inproceedings{banarjee2005, + title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, + author = {Banerjee, Satanjeev and Lavie, Alon}, + booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, + month = jun, + year = {2005}, + address = {Ann Arbor, Michigan}, + publisher = {Association for Computational Linguistics}, + url = {https://www.aclweb.org/anthology/W05-0909}, + pages = {65--72}, +} +""" + +_DESCRIPTION = """\ +METEOR, an automatic metric for machine translation evaluation +that is based on a generalized concept of unigram matching between the +machine-produced translation and human-produced reference translations. +Unigrams can be matched based on their surface forms, stemmed forms, +and meanings; furthermore, METEOR can be easily extended to include more +advanced matching strategies. Once all generalized unigram matches +between the two strings have been found, METEOR computes a score for +this matching using a combination of unigram-precision, unigram-recall, and +a measure of fragmentation that is designed to directly capture how +well-ordered the matched words in the machine translation are in relation +to the reference. + +METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic +data and 0.331 on the Chinese data. This is shown to be an improvement on +using simply unigram-precision, unigram-recall and their harmonic F1 +combination. +""" + +_KWARGS_DESCRIPTION = """ +Computes METEOR score of translated segments against one or more references. +Args: + predictions: list of predictions to score. Each prediction + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. + alpha: Parameter for controlling relative weights of precision and recall. default: 0.9 + beta: Parameter for controlling shape of penalty as a function of fragmentation. default: 3 + gamma: Relative weight assigned to fragmentation penalty. default: 0.5 +Returns: + 'meteor': meteor score. +Examples: + + >>> meteor = evaluate.load('meteor') + >>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] + >>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] + >>> results = meteor.compute(predictions=predictions, references=references) + >>> print(round(results["meteor"], 4)) + 0.6944 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Meteor(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py"], + reference_urls=[ + "https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score", + "https://en.wikipedia.org/wiki/METEOR", + ], + ) + + def _download_and_prepare(self, dl_manager): + import nltk + + nltk.download("wordnet") + if NLTK_VERSION >= version.Version("3.6.5"): + nltk.download("punkt") + if NLTK_VERSION >= version.Version("3.6.6"): + nltk.download("omw-1.4") + + def _compute(self, predictions, references, alpha=0.9, beta=3, gamma=0.5): + multiple_refs = isinstance(references[0], list) + if NLTK_VERSION >= version.Version("3.6.5"): + # the version of METEOR in NLTK version 3.6.5 and earlier expect tokenized inputs + if multiple_refs: + scores = [ + meteor_score.meteor_score( + [word_tokenize(ref) for ref in refs], + word_tokenize(pred), + alpha=alpha, + beta=beta, + gamma=gamma, + ) + for refs, pred in zip(references, predictions) + ] + else: + scores = [ + meteor_score.single_meteor_score( + word_tokenize(ref), word_tokenize(pred), alpha=alpha, beta=beta, gamma=gamma + ) + for ref, pred in zip(references, predictions) + ] + else: + if multiple_refs: + scores = [ + meteor_score.meteor_score( + [[word_tokenize(ref) for ref in group] for group in references][0], + word_tokenize(pred), + alpha=alpha, + beta=beta, + gamma=gamma, + ) + for ref, pred in zip(references, predictions) + ] + else: + scores = [ + meteor_score.single_meteor_score(ref, pred, alpha=alpha, beta=beta, gamma=gamma) + for ref, pred in zip(references, predictions) + ] + + return {"meteor": np.mean(scores)} diff --git a/ST/evaluate/metrics/meteor/requirements.txt b/ST/evaluate/metrics/meteor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5904a71469bea16b7ccb3b41e9ed19663d20599 --- /dev/null +++ b/ST/evaluate/metrics/meteor/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk \ No newline at end of file diff --git a/ST/evaluate/metrics/mse/README.md b/ST/evaluate/metrics/mse/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6f4144ec3fc649340ce77ecddc60a43328c907c6 --- /dev/null +++ b/ST/evaluate/metrics/mse/README.md @@ -0,0 +1,137 @@ +--- +title: MSE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Squared Error(MSE) is the average of the square of difference between the predicted + and actual values. +--- + +# Metric Card for MSE + + +## Metric Description + +Mean Squared Error(MSE) represents the average of the squares of errors -- i.e. the average squared difference between the estimated values and the actual values. + +![image](https://user-images.githubusercontent.com/14205986/165999302-eba3702d-81e3-4363-9c0e-d3bfceb7ec5a.png) + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mse_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. +- `squared` (`bool`): If `True` returns MSE value, if `False` returns RMSE (Root Mean Squared Error). The default value is `True`. + + +### Output Values +This metric outputs a dictionary, containing the mean squared error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MSE `float` value ranges from `0.0` to `1.0`, with the best value being `0.0`. + +Output Example(s): +```python +{'mse': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mse': array([0.41666667, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mse_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mse': 0.375} +``` + +Example with `squared = True`, which returns the RMSE: +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) +>>> print(rmse_result) +{'mse': 0.6123724356957945} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mse_metric = evaluate.load("mse", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0, 2], [-1, 2], [8, -5]] +>>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mse': array([0.41666667, 1. ])} +""" +``` + +## Limitations and Bias +MSE has the disadvantage of heavily weighting outliers -- given that it squares them, this results in large errors weighing more heavily than small ones. It can be used alongside [MAE](https://huggingface.co/metrics/mae), which is complementary given that it does not square the errors. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{willmott2005advantages, + title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, + author={Willmott, Cort J and Matsuura, Kenji}, + journal={Climate research}, + volume={30}, + number={1}, + pages={79--82}, + year={2005} +} +``` + +## Further References +- [Mean Squared Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_squared_error) diff --git a/ST/evaluate/metrics/mse/app.py b/ST/evaluate/metrics/mse/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d334862deefa700035cdcd573ba00e6b4304121b --- /dev/null +++ b/ST/evaluate/metrics/mse/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mse") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/mse/mse.py b/ST/evaluate/metrics/mse/mse.py new file mode 100644 index 0000000000000000000000000000000000000000..fb695bfde89e38d17dc1761197768e8cc8061b33 --- /dev/null +++ b/ST/evaluate/metrics/mse/mse.py @@ -0,0 +1,119 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MSE - Mean Squared Error Metric""" + +import datasets +from sklearn.metrics import mean_squared_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Squared Error(MSE) is the average of the square of difference between the predicted +and actual values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + + squared : bool, default=True + If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value. + +Returns: + mse : mean squared error. +Examples: + + >>> mse_metric = evaluate.load("mse") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mse_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mse': 0.375} + >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) + >>> print(rmse_result) + {'mse': 0.6123724356957945} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mse_metric = evaluate.load("mse", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0, 2], [-1, 2], [8, -5]] + >>> results = mse_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mse': 0.7083333333333334} + >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) # doctest: +NORMALIZE_WHITESPACE + {'mse': array([0.41666667, 1. ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mse(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average", squared=True): + + mse = mean_squared_error( + references, predictions, sample_weight=sample_weight, multioutput=multioutput, squared=squared + ) + + return {"mse": mse} diff --git a/ST/evaluate/metrics/mse/requirements.txt b/ST/evaluate/metrics/mse/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/mse/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/nist_mt/README.md b/ST/evaluate/metrics/nist_mt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ee553a32afab574ad2813d1d8fc0ade04e32b7db --- /dev/null +++ b/ST/evaluate/metrics/nist_mt/README.md @@ -0,0 +1,82 @@ +--- +title: NIST_MT +emoji: 🤗 +colorFrom: purple +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: + DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU score. +--- + +# Metric Card for NIST's MT metric + + +## Metric Description +DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU +score. The official script used by NIST to compute BLEU and NIST score is +mteval-14.pl. The main differences are: + + - BLEU uses geometric mean of the ngram overlaps, NIST uses arithmetic mean. + - NIST has a different brevity penalty + - NIST score from mteval-14.pl has a self-contained tokenizer (in the Hugging Face implementation we rely on NLTK's +implementation of the NIST-specific tokenizer) + +## Intended Uses +NIST was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +nist_mt = evaluate.load("nist_mt") +hypothesis1 = "It is a guide to action which ensures that the military always obeys the commands of the party" +reference1 = "It is a guide to action that ensures that the military will forever heed Party commands" +reference2 = "It is the guiding principle which guarantees the military forces always being under the command of the Party" +nist_mt.compute(hypothesis1, [reference1, reference2]) +# {'nist_mt': 3.3709935957649324} +``` + +### Inputs +- **predictions**: tokenized predictions to score. For sentence-level NIST, a list of tokens (str); + for corpus-level NIST, a list (sentences) of lists of tokens (str) +- **references**: potentially multiple tokenized references for each prediction. For sentence-level NIST, a + list (multiple potential references) of list of tokens (str); for corpus-level NIST, a list (corpus) of lists + (multiple potential references) of lists of tokens (str) +- **n**: highest n-gram order +- **tokenize_kwargs**: arguments passed to the tokenizer (see: https://github.com/nltk/nltk/blob/90fa546ea600194f2799ee51eaf1b729c128711e/nltk/tokenize/nist.py#L139) + +### Output Values +- **nist_mt** (`float`): NIST score + +Output Example: +```python +{'nist_mt': 3.3709935957649324} +``` + + +## Citation +```bibtex +@inproceedings{10.5555/1289189.1289273, + author = {Doddington, George}, + title = {Automatic Evaluation of Machine Translation Quality Using N-Gram Co-Occurrence Statistics}, + year = {2002}, + publisher = {Morgan Kaufmann Publishers Inc.}, + address = {San Francisco, CA, USA}, + booktitle = {Proceedings of the Second International Conference on Human Language Technology Research}, + pages = {138–145}, + numpages = {8}, + location = {San Diego, California}, + series = {HLT '02} +} +``` + +## Further References + +This Hugging Face implementation uses [the NLTK implementation](https://github.com/nltk/nltk/blob/develop/nltk/translate/nist_score.py) diff --git a/ST/evaluate/metrics/nist_mt/app.py b/ST/evaluate/metrics/nist_mt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0a563648a5b9a8624ecd508248c9a15be8dbba97 --- /dev/null +++ b/ST/evaluate/metrics/nist_mt/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("nist_mt") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/nist_mt/nist_mt.py b/ST/evaluate/metrics/nist_mt/nist_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..5aeab0c08bd75f3039ab8b7452adf52d0979b808 --- /dev/null +++ b/ST/evaluate/metrics/nist_mt/nist_mt.py @@ -0,0 +1,132 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NLTK's NIST implementation on both the sentence and corpus level""" +from typing import Dict, Optional + +import datasets +import nltk +from datasets import Sequence, Value + + +try: + nltk.data.find("perluniprops") +except LookupError: + nltk.download("perluniprops", quiet=True) # NISTTokenizer requirement + +from nltk.tokenize.nist import NISTTokenizer +from nltk.translate.nist_score import corpus_nist, sentence_nist + +import evaluate + + +_CITATION = """\ +@inproceedings{10.5555/1289189.1289273, + author = {Doddington, George}, + title = {Automatic Evaluation of Machine Translation Quality Using N-Gram Co-Occurrence Statistics}, + year = {2002}, + publisher = {Morgan Kaufmann Publishers Inc.}, + address = {San Francisco, CA, USA}, + booktitle = {Proceedings of the Second International Conference on Human Language Technology Research}, + pages = {138–145}, + numpages = {8}, + location = {San Diego, California}, + series = {HLT '02} +} +""" + +_DESCRIPTION = """\ +DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU +score. The official script used by NIST to compute BLEU and NIST score is +mteval-14.pl. The main differences are: + + - BLEU uses geometric mean of the ngram precisions, NIST uses arithmetic mean. + - NIST has a different brevity penalty + - NIST score from mteval-14.pl has a self-contained tokenizer (in the Hugging Face implementation we rely on NLTK's +implementation of the NIST-specific tokenizer) +""" + + +_KWARGS_DESCRIPTION = """ +Computes NIST score of translated segments against one or more references. +Args: + predictions: predictions to score (list of str) + references: potentially multiple references for each prediction (list of str or list of list of str) + n: highest n-gram order + lowercase: whether to lowercase the data (only applicable if 'western_lang' is True) + western_lang: whether the current language is a Western language, which will enable some specific tokenization + rules with respect to, e.g., punctuation + +Returns: + 'nist_mt': nist_mt score +Examples: + >>> nist_mt = evaluate.load("nist_mt") + >>> hypothesis = "It is a guide to action which ensures that the military always obeys the commands of the party" + >>> reference1 = "It is a guide to action that ensures that the military will forever heed Party commands" + >>> reference2 = "It is the guiding principle which guarantees the military forces always being under the command of the Party" + >>> reference3 = "It is the practical guide for the army always to heed the directions of the party" + >>> nist_mt.compute(predictions=[hypothesis], references=[[reference1, reference2, reference3]]) + {'nist_mt': 3.3709935957649324} + >>> nist_mt.compute(predictions=[hypothesis], references=[reference1]) + {'nist_mt': 2.4477124183006533} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class NistMt(evaluate.Metric): + """A wrapper around NLTK's NIST implementation.""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Sequence(Value("string", id="reference"), id="references"), + } + ), + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Value("string", id="reference"), + } + ), + ], + homepage="https://www.nltk.org/api/nltk.translate.nist_score.html", + codebase_urls=["https://github.com/nltk/nltk/blob/develop/nltk/translate/nist_score.py"], + reference_urls=["https://en.wikipedia.org/wiki/NIST_(metric)"], + ) + + def _compute(self, predictions, references, n: int = 5, lowercase=False, western_lang=True): + tokenizer = NISTTokenizer() + + # Account for single reference cases: references always need to have one more dimension than predictions + if isinstance(references[0], str): + references = [[ref] for ref in references] + + predictions = [ + tokenizer.tokenize(pred, return_str=False, lowercase=lowercase, western_lang=western_lang) + for pred in predictions + ] + references = [ + [ + tokenizer.tokenize(ref, return_str=False, lowercase=lowercase, western_lang=western_lang) + for ref in ref_sentences + ] + for ref_sentences in references + ] + return {"nist_mt": corpus_nist(list_of_references=references, hypotheses=predictions, n=n)} diff --git a/ST/evaluate/metrics/nist_mt/requirements.txt b/ST/evaluate/metrics/nist_mt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5844221307e93baf24d64fb2b6df6622b2375a --- /dev/null +++ b/ST/evaluate/metrics/nist_mt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk diff --git a/ST/evaluate/metrics/nist_mt/tests.py b/ST/evaluate/metrics/nist_mt/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..525287274f6137b655c504082debdd10eae5f982 --- /dev/null +++ b/ST/evaluate/metrics/nist_mt/tests.py @@ -0,0 +1,34 @@ +from _pytest.fixtures import fixture +from nist_mt import Nist_mt + + +nist = Nist_mt() + + +@fixture +def hypothesis_sent(): + return "It is a guide to action which ensures that the military always obeys the commands of the party" + + +@fixture +def reference_sent1(): + return "It is a guide to action that ensures that the military will forever heed Party commands" + + +@fixture +def reference_sent2(): + return ( + "It is the guiding principle which guarantees the military forces always being under the command of the Party" + ) + + +@fixture +def reference_sent3(): + return "It is the practical guide for the army always to heed the directions of the party" + + +def test_nist_sentence(hypothesis_sent, reference_sent1, reference_sent2, reference_sent3): + nist_score = nist.compute( + predictions=[hypothesis_sent], references=[[reference_sent1, reference_sent2, reference_sent3]] + ) + assert abs(nist_score["nist_mt"] - 3.3709935957649324) < 1e-6 diff --git a/ST/evaluate/metrics/pearsonr/README.md b/ST/evaluate/metrics/pearsonr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..391d65267014015c0f84ce91ab30a982e6bf6bac --- /dev/null +++ b/ST/evaluate/metrics/pearsonr/README.md @@ -0,0 +1,119 @@ +--- +title: Pearson Correlation Coefficient +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Pearson correlation coefficient and p-value for testing non-correlation. + The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. + The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. +--- + +# Metric Card for Pearson Correlation Coefficient (pearsonr) + + +## Metric Description + +Pearson correlation coefficient and p-value for testing non-correlation. +The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. +The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. + + +## How to Use + +This metric takes a list of predictions and a list of references as input + +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) +>>> print(round(results['pearsonr']), 2) +['-0.74'] +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted class labels, as returned by a model. +- **references** (`list` of `int`): Ground truth labels. +- **return_pvalue** (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. + + +### Output Values +- **pearsonr**(`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. +- **p-value**(`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. + +Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. + +Output Example(s): +```python +{'pearsonr': -0.7} +``` +```python +{'p-value': 0.15} +``` + +#### Values from Popular Papers + +### Examples + +Example 1-A simple example using only predictions and references. +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) +>>> print(round(results['pearsonr'], 2)) +-0.74 +``` + +Example 2-The same as Example 1, but that also returns the `p-value`. +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) +>>> print(sorted(list(results.keys()))) +['p-value', 'pearsonr'] +>>> print(round(results['pearsonr'], 2)) +-0.74 +>>> print(round(results['p-value'], 2)) +0.15 +``` + + +## Limitations and Bias + +As stated above, the calculation of the p-value relies on the assumption that each data set is normally distributed. This is not always the case, so verifying the true distribution of datasets is recommended. + + +## Citation(s) +```bibtex +@article{2020SciPy-NMeth, +author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, +title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, +journal = {Nature Methods}, +year = {2020}, +volume = {17}, +pages = {261--272}, +adsurl = {https://rdcu.be/b08Wh}, +doi = {10.1038/s41592-019-0686-2}, +} +``` + + +## Further References diff --git a/ST/evaluate/metrics/pearsonr/app.py b/ST/evaluate/metrics/pearsonr/app.py new file mode 100644 index 0000000000000000000000000000000000000000..2123f9f62a2ae6a5f2cb772b2c92e05f4bc03f90 --- /dev/null +++ b/ST/evaluate/metrics/pearsonr/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("pearsonr") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/pearsonr/pearsonr.py b/ST/evaluate/metrics/pearsonr/pearsonr.py new file mode 100644 index 0000000000000000000000000000000000000000..5ed0e76206f2876d52106eb9fd01185ab2bb41fd --- /dev/null +++ b/ST/evaluate/metrics/pearsonr/pearsonr.py @@ -0,0 +1,107 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pearson correlation coefficient metric.""" + +import datasets +from scipy.stats import pearsonr + +import evaluate + + +_DESCRIPTION = """ +Pearson correlation coefficient and p-value for testing non-correlation. +The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. +The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted class labels, as returned by a model. + references (`list` of `int`): Ground truth labels. + return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. + +Returns: + pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. + p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. + +Examples: + + Example 1-A simple example using only predictions and references. + >>> pearsonr_metric = evaluate.load("pearsonr") + >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) + >>> print(round(results['pearsonr'], 2)) + -0.74 + + Example 2-The same as Example 1, but that also returns the `p-value`. + >>> pearsonr_metric = evaluate.load("pearsonr") + >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) + >>> print(sorted(list(results.keys()))) + ['p-value', 'pearsonr'] + >>> print(round(results['pearsonr'], 2)) + -0.74 + >>> print(round(results['p-value'], 2)) + 0.15 +""" + + +_CITATION = """ +@article{2020SciPy-NMeth, +author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, Ilhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Antonio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, +title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, +journal = {Nature Methods}, +year = {2020}, +volume = {17}, +pages = {261--272}, +adsurl = {https://rdcu.be/b08Wh}, +doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Pearsonr(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + ), + reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"], + ) + + def _compute(self, predictions, references, return_pvalue=False): + if return_pvalue: + results = pearsonr(references, predictions) + return {"pearsonr": results[0], "p-value": results[1]} + else: + return {"pearsonr": float(pearsonr(references, predictions)[0])} diff --git a/ST/evaluate/metrics/pearsonr/requirements.txt b/ST/evaluate/metrics/pearsonr/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/ST/evaluate/metrics/pearsonr/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/ST/evaluate/metrics/perplexity/README.md b/ST/evaluate/metrics/perplexity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7948097f8903b2e7f6fef3ae36f8b6153991cc1d --- /dev/null +++ b/ST/evaluate/metrics/perplexity/README.md @@ -0,0 +1,118 @@ +--- +title: Perplexity +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Perplexity (PPL) is one of the most common metrics for evaluating language models. + It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + + For more information on perplexity, see [this tutorial](https://huggingface.co/docs/transformers/perplexity). +--- + +# Metric Card for Perplexity + +## Metric Description +Given a model and an input text sequence, perplexity measures how likely the model is to generate the input text sequence. + +As a metric, it can be used to evaluate how well the model has learned the distribution of the text it was trained on. + +In this case, `model_id` should be the trained model to be evaluated, and the input texts should be the text that the model was trained on. + +This implementation of perplexity is calculated with log base `e`, as in `perplexity = e**(sum(losses) / num_tokenized_tokens)`, following recent convention in deep learning frameworks. + +## Intended Uses +Any language generation task. + +## How to Use + +The metric takes a list of text as input, as well as the name of the model used to compute the metric: + +```python +from evaluate import load +perplexity = load("perplexity", module_type="metric") +results = perplexity.compute(predictions=predictions, model_id='gpt2') +``` + +### Inputs +- **model_id** (str): model used for calculating Perplexity. NOTE: Perplexity can only be calculated for causal language models. + - This includes models such as gpt2, causal variations of bert, causal versions of t5, and more (the full list can be found in the AutoModelForCausalLM documentation here: https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) +- **predictions** (list of str): input text, where each separate text snippet is one list entry. +- **batch_size** (int): the batch size to run texts through the model. Defaults to 16. +- **add_start_token** (bool): whether to add the start token to the texts, so the perplexity can include the probability of the first word. Defaults to True. +- **device** (str): device to run on, defaults to `cuda` when available + +### Output Values +This metric outputs a dictionary with the perplexity scores for the text input in the list, and the average perplexity. +If one of the input texts is longer than the max input length of the model, then it is truncated to the max length for the perplexity computation. + +``` +{'perplexities': [8.182524681091309, 33.42122268676758, 27.012239456176758], 'mean_perplexity': 22.871995608011883} +``` + +The range of this metric is [0, inf). A lower score is better. + +#### Values from Popular Papers + + +### Examples +Calculating perplexity on predictions defined here: +```python +perplexity = evaluate.load("perplexity", module_type="metric") +input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] +results = perplexity.compute(model_id='gpt2', + add_start_token=False, + predictions=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>646.75 +print(round(results["perplexities"][0], 2)) +>>>32.25 +``` +Calculating perplexity on predictions loaded in from a dataset: +```python +perplexity = evaluate.load("perplexity", module_type="metric") +input_texts = datasets.load_dataset("wikitext", + "wikitext-2-raw-v1", + split="test")["text"][:50] +input_texts = [s for s in input_texts if s!=''] +results = perplexity.compute(model_id='gpt2', + predictions=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>576.76 +print(round(results["perplexities"][0], 2)) +>>>889.28 +``` + +## Limitations and Bias +Note that the output value is based heavily on what text the model was trained on. This means that perplexity scores are not comparable between models or datasets. + +See Meister and Cotterell, ["Language Model Evaluation Beyond Perplexity"]( https://arxiv.org/abs/2106.00085) (2021) for more information about alternative model evaluation strategies. + +## Citation + +```bibtex +@article{jelinek1977perplexity, +title={Perplexity—a measure of the difficulty of speech recognition tasks}, +author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K}, +journal={The Journal of the Acoustical Society of America}, +volume={62}, +number={S1}, +pages={S63--S63}, +year={1977}, +publisher={Acoustical Society of America} +} +``` + +## Further References +- [Hugging Face Perplexity Blog Post](https://huggingface.co/docs/transformers/perplexity) diff --git a/ST/evaluate/metrics/perplexity/app.py b/ST/evaluate/metrics/perplexity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..cf4eee44294f2c28e9aea9adc0b7f8abedfb2e0b --- /dev/null +++ b/ST/evaluate/metrics/perplexity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("perplexity", module_type="metric") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/perplexity/perplexity.py b/ST/evaluate/metrics/perplexity/perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..ad307e8ad5cf09d8cfd19390ad539f24187631d0 --- /dev/null +++ b/ST/evaluate/metrics/perplexity/perplexity.py @@ -0,0 +1,192 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Perplexity Metric.""" + +import datasets +import numpy as np +import torch +from torch.nn import CrossEntropyLoss +from transformers import AutoModelForCausalLM, AutoTokenizer + +import evaluate +from evaluate import logging + + +_CITATION = """\ + +""" + +_DESCRIPTION = """ +Perplexity (PPL) is one of the most common metrics for evaluating language models. +It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + +For more information, see https://huggingface.co/docs/transformers/perplexity +""" + +_KWARGS_DESCRIPTION = """ +Args: + model_id (str): model used for calculating Perplexity + NOTE: Perplexity can only be calculated for causal language models. + This includes models such as gpt2, causal variations of bert, + causal versions of t5, and more (the full list can be found + in the AutoModelForCausalLM documentation here: + https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) + + predictions (list of str): input text, each separate text snippet + is one list entry. + batch_size (int): the batch size to run texts through the model. Defaults to 16. + add_start_token (bool): whether to add the start token to the texts, + so the perplexity can include the probability of the first word. Defaults to True. + device (str): device to run on, defaults to 'cuda' when available +Returns: + perplexity: dictionary containing the perplexity scores for the texts + in the input list, as well as the mean perplexity. If one of the input texts is + longer than the max input length of the model, then it is truncated to the + max length for the perplexity computation. +Examples: + Example 1: + >>> perplexity = evaluate.load("perplexity", module_type="metric") + >>> input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] + >>> results = perplexity.compute(model_id='gpt2', + ... add_start_token=False, + ... predictions=input_texts) # doctest:+ELLIPSIS + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 0)) + 647.0 + >>> print(round(results["perplexities"][0], 0)) + 32.0 + + Example 2: + >>> from datasets import load_dataset + >>> perplexity = evaluate.load("perplexity", module_type="metric") + >>> input_texts = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] # doctest: +SKIP + >>> input_texts = [s for s in input_texts if s!=''] + >>> results = perplexity.compute(model_id='gpt2', + ... predictions=input_texts) + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 2)) # doctest: +SKIP + 576.76 + >>> print(round(results["perplexities"][0], 2)) # doctest: +SKIP + 889.28 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Perplexity(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + } + ), + reference_urls=["https://huggingface.co/docs/transformers/perplexity"], + ) + + def _compute( + self, predictions, model_id, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None + ): + + if device is not None: + assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu." + if device == "gpu": + device = "cuda" + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + + model = AutoModelForCausalLM.from_pretrained(model_id) + model = model.to(device) + + tokenizer = AutoTokenizer.from_pretrained(model_id) + + # if batch_size > 1 (which generally leads to padding being required), and + # if there is not an already assigned pad_token, assign an existing + # special token to also be the padding token + if tokenizer.pad_token is None and batch_size > 1: + existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) + # check that the model already has at least one special token defined + assert ( + len(existing_special_tokens) > 0 + ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." + # assign one of the special tokens to also be the pad token + tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]}) + + if add_start_token and max_length: + # leave room for token to be added: + assert ( + tokenizer.bos_token is not None + ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False" + max_tokenized_len = max_length - 1 + else: + max_tokenized_len = max_length + + encodings = tokenizer( + predictions, + add_special_tokens=False, + padding=True, + truncation=True if max_tokenized_len else False, + max_length=max_tokenized_len, + return_tensors="pt", + return_attention_mask=True, + ).to(device) + + encoded_texts = encodings["input_ids"] + attn_masks = encodings["attention_mask"] + + # check that each input is long enough: + if add_start_token: + assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long." + else: + assert torch.all( + torch.ge(attn_masks.sum(1), 2) + ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings." + + ppls = [] + loss_fct = CrossEntropyLoss(reduction="none") + + for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)): + end_index = min(start_index + batch_size, len(encoded_texts)) + encoded_batch = encoded_texts[start_index:end_index] + attn_mask = attn_masks[start_index:end_index] + + if add_start_token: + bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device) + encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1) + attn_mask = torch.cat( + [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1 + ) + + labels = encoded_batch + + with torch.no_grad(): + out_logits = model(encoded_batch, attention_mask=attn_mask).logits + + shift_logits = out_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_attention_mask_batch = attn_mask[..., 1:].contiguous() + + perplexity_batch = torch.exp( + (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1) + / shift_attention_mask_batch.sum(1) + ) + + ppls += perplexity_batch.tolist() + + return {"perplexities": ppls, "mean_perplexity": np.mean(ppls)} diff --git a/ST/evaluate/metrics/perplexity/requirements.txt b/ST/evaluate/metrics/perplexity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..036c7211c00774608a613a2354aaae763c03bbb5 --- /dev/null +++ b/ST/evaluate/metrics/perplexity/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +torch +transformers \ No newline at end of file diff --git a/ST/evaluate/metrics/poseval/README.md b/ST/evaluate/metrics/poseval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..470300f2e1f3c0fc9114d9c95c029ed0919e14ce --- /dev/null +++ b/ST/evaluate/metrics/poseval/README.md @@ -0,0 +1,117 @@ +--- +title: poseval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data + that is not in IOB format the poseval is an alternative. It treats each token in the dataset as independant + observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's + classification report to compute the scores. +--- + +# Metric Card for peqeval + +## Metric description + +The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data (see e.g. [here](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging)) that is not in IOB format the poseval is an alternative. It treats each token in the dataset as independant observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's [classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) to compute the scores. + + +## How to use + +Poseval produces labelling scores along with its sufficient statistics from a source against references. + +It takes two mandatory arguments: + +`predictions`: a list of lists of predicted labels, i.e. estimated targets as returned by a tagger. + +`references`: a list of lists of reference labels, i.e. the ground truth/target values. + +It can also take several optional arguments: + +`zero_division`: Which value to substitute as a metric value when encountering zero division. Should be one of [`0`,`1`,`"warn"`]. `"warn"` acts as `0`, but the warning is raised. + + +```python +>>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] +>>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] +>>> poseval = evaluate.load("poseval") +>>> results = poseval.compute(predictions=predictions, references=references) +>>> print(list(results.keys())) +['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] +>>> print(results["accuracy"]) +0.8 +>>> print(results["PROPN"]["recall"]) +0.5 +``` + +## Output values + +This metric returns a a classification report as a dictionary with a summary of scores for overall and per type: + +Overall (weighted and macro avg): + +`accuracy`: the average [accuracy](https://huggingface.co/metrics/accuracy), on a scale between 0.0 and 1.0. + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), which is the harmonic mean of the precision and recall. It also has a scale of 0.0 to 1.0. + +Per type (e.g. `MISC`, `PER`, `LOC`,...): + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), on a scale between 0.0 and 1.0. + + +## Examples + +```python +>>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] +>>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] +>>> poseval = evaluate.load("poseval") +>>> results = poseval.compute(predictions=predictions, references=references) +>>> print(list(results.keys())) +['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] +>>> print(results["accuracy"]) +0.8 +>>> print(results["PROPN"]["recall"]) +0.5 +``` + +## Limitations and bias + +In contrast to [seqeval](https://github.com/chakki-works/seqeval), the poseval metric treats each token independently and computes the classification report over all concatenated sequences.. + + +## Citation + +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +## Further References +- [README for seqeval at GitHub](https://github.com/chakki-works/seqeval) +- [Classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) +- [Issues with seqeval](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging) \ No newline at end of file diff --git a/ST/evaluate/metrics/poseval/app.py b/ST/evaluate/metrics/poseval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..bf11e456a33c0cf6fcacbe5ff0ff20aed4514e32 --- /dev/null +++ b/ST/evaluate/metrics/poseval/app.py @@ -0,0 +1,7 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("poseval") + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/poseval/poseval.py b/ST/evaluate/metrics/poseval/poseval.py new file mode 100644 index 0000000000000000000000000000000000000000..124146cd024c05e01116403d6c4a164165288bd3 --- /dev/null +++ b/ST/evaluate/metrics/poseval/poseval.py @@ -0,0 +1,113 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" seqeval metric. """ + +from typing import Union + +import datasets +from sklearn.metrics import classification_report + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data \ +(see e.g. [here](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging))\ +that is not in IOB format the poseval metric is an alternative. It treats each token in the dataset as independant \ +observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's \ +[classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) \ +to compute the scores. + +""" + +_KWARGS_DESCRIPTION = """ +Computes the poseval metric. + +Args: + predictions: List of List of predicted labels (Estimated targets as returned by a tagger) + references: List of List of reference labels (Ground truth (correct) target values) + zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + 'scores': dict. Summary of the scores for overall and per type + Overall (weighted and macro avg): + 'accuracy': accuracy, + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure, + Per type: + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure +Examples: + + >>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] + >>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] + >>> poseval = evaluate.load("poseval") + >>> results = poseval.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] + >>> print(results["accuracy"]) + 0.8 + >>> print(results["PROPN"]["recall"]) + 0.5 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Poseval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://scikit-learn.org", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + } + ), + codebase_urls=["https://github.com/scikit-learn/scikit-learn"], + ) + + def _compute( + self, + predictions, + references, + zero_division: Union[str, int] = "warn", + ): + report = classification_report( + y_true=[label for ref in references for label in ref], + y_pred=[label for pred in predictions for label in pred], + output_dict=True, + zero_division=zero_division, + ) + + return report diff --git a/ST/evaluate/metrics/poseval/requirements.txt b/ST/evaluate/metrics/poseval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/poseval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/precision/README.md b/ST/evaluate/metrics/precision/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a88670c72854c493656d8174ab5bec02ea28fbb7 --- /dev/null +++ b/ST/evaluate/metrics/precision/README.md @@ -0,0 +1,142 @@ +--- +title: Precision +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: + Precision = TP / (TP + FP) + where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). +--- + +# Metric Card for Precision + + +## Metric Description + +Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: +Precision = TP / (TP + FP) +where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). + + +## How to Use + +At minimum, precision takes as input a list of predicted labels, `predictions`, and a list of output labels, `references`. + +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'precision': 1.0} +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted class labels. +- **references** (`list` of `int`): Actual class labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - 0: Returns 0 when there is a zero division. + - 1: Returns 1 when there is a zero division. + - 'warn': Raises warnings and then returns 0 when there is a zero division. + + +### Output Values +- **precision**(`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. + +Output Example(s): +```python +{'precision': 0.2222222222222222} +``` +```python +{'precision': array([0.66666667, 0.0, 0.0])} +``` + + + + +#### Values from Popular Papers + + +### Examples + +Example 1-A simple binary example +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) +>>> print(results) +{'precision': 0.5} +``` + +Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) +>>> print(round(results['precision'], 2)) +0.67 +``` + +Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) +>>> print(results) +{'precision': 0.23529411764705882} +``` + +Example 4-A multiclass example, with different values for the `average` input. +```python +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') +>>> print(results) +{'precision': 0.2222222222222222} +>>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') +>>> print(results) +{'precision': 0.3333333333333333} +>>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') +>>> print(results) +{'precision': 0.2222222222222222} +>>> results = precision_metric.compute(predictions=predictions, references=references, average=None) +>>> print([round(res, 2) for res in results['precision']]) +[0.67, 0.0, 0.0] +``` + + +## Limitations and Bias + +[Precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall) are complementary and can be used to measure different aspects of model performance -- using both of them (or an averaged measure like [F1 score](https://huggingface.co/metrics/F1) to better represent different aspects of performance. See [Wikipedia](https://en.wikipedia.org/wiki/Precision_and_recall) for more information. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References +- [Wikipedia -- Precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) diff --git a/ST/evaluate/metrics/precision/app.py b/ST/evaluate/metrics/precision/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ec80edf5a54da759b8e1509650826f0a8e8b8bd7 --- /dev/null +++ b/ST/evaluate/metrics/precision/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("precision") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/precision/precision.py b/ST/evaluate/metrics/precision/precision.py new file mode 100644 index 0000000000000000000000000000000000000000..4b35aa7e44f262ec9bbc83500bc55471beb465bd --- /dev/null +++ b/ST/evaluate/metrics/precision/precision.py @@ -0,0 +1,145 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Precision metric.""" + +import datasets +from sklearn.metrics import precision_score + +import evaluate + + +_DESCRIPTION = """ +Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: +Precision = TP / (TP + FP) +where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted class labels. + references (`list` of `int`): Actual class labels. + labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. + pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. + average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). + sample_weight (`list` of `float`): Sample weights Defaults to None. + zero_division (`int` or `string`): Sets the value to return when there is a zero division. Defaults to 'warn'. + + - 0: Returns 0 when there is a zero division. + - 1: Returns 1 when there is a zero division. + - 'warn': Raises warnings and then returns 0 when there is a zero division. + +Returns: + precision (`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. + +Examples: + + Example 1-A simple binary example + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) + >>> print(results) + {'precision': 0.5} + + Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) + >>> print(round(results['precision'], 2)) + 0.67 + + Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) + >>> print(results) + {'precision': 0.23529411764705882} + + Example 4-A multiclass example, with different values for the `average` input. + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') + >>> print(results) + {'precision': 0.2222222222222222} + >>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') + >>> print(results) + {'precision': 0.3333333333333333} + >>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') + >>> print(results) + {'precision': 0.2222222222222222} + >>> results = precision_metric.compute(predictions=predictions, references=references, average=None) + >>> print([round(res, 2) for res in results['precision']]) + [0.67, 0.0, 0.0] +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Precision(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html"], + ) + + def _compute( + self, + predictions, + references, + labels=None, + pos_label=1, + average="binary", + sample_weight=None, + zero_division="warn", + ): + score = precision_score( + references, + predictions, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division, + ) + return {"precision": float(score) if score.size == 1 else score} diff --git a/ST/evaluate/metrics/precision/requirements.txt b/ST/evaluate/metrics/precision/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/precision/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/r_squared/README.md b/ST/evaluate/metrics/r_squared/README.md new file mode 100644 index 0000000000000000000000000000000000000000..00c97c6d6a8c1d63ced5ab59047577ab8174ec55 --- /dev/null +++ b/ST/evaluate/metrics/r_squared/README.md @@ -0,0 +1,86 @@ +--- +title: r_squared +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The R^2 (R Squared) metric is a measure of the goodness of fit of a linear regression model. It is the proportion of the variance in the dependent variable that is predictable from the independent variable. +--- + +# Metric Card for R^2 + +## Metric description + +An R-squared value of 1 indicates that the model perfectly explains the variance of the dependent variable. A value of 0 means that the model does not explain any of the variance. Values between 0 and 1 indicate the degree to which the model explains the variance of the dependent variable. + +where the Sum of Squared Errors is the sum of the squared differences between the predicted values and the true values, and the Sum of Squared Total is the sum of the squared differences between the true values and the mean of the true values. + +For example, if an R-squared value for a model is 0.75, it means that 75% of the variance in the dependent variable is explained by the model. + +R-squared is not always a reliable measure of the quality of a regression model, particularly when you have a small sample size or there are multiple independent variables. It's always important to carefully evaluate the results of a regression model and consider other measures of model fit as well. + +R squared can be calculated using the following formula: + +```python +r_squared = 1 - (Sum of Squared Errors / Sum of Squared Total) +``` + +* Calculate the residual sum of squares (RSS), which is the sum of the squared differences between the predicted values and the actual values. +* Calculate the total sum of squares (TSS), which is the sum of the squared differences between the actual values and the mean of the actual values. +* Calculate the R-squared value by taking 1 - (RSS / TSS). + +Here's an example of how to calculate the R-squared value: +```python +r_squared = 1 - (SSR/SST) +``` + +### How to Use Examples: + +The R2 class in the evaluate module can be used to compute the R^2 value for a given set of predictions and references. (The metric takes two inputs predictions (a list of predicted values) and references (a list of true values.)) + +```python +from evaluate import load +>>> r2_metric = evaluate.load("r_squared") +>>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[0.9, 2.1, 3.2, 3.8]) +>>> print(r_squared) +0.98 +``` + +Alternatively, if you want to see an example where there is a perfect match between the prediction and reference: +```python +>>> from evaluate import load +>>> r2_metric = evaluate.load("r_squared") +>>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[1, 2, 3, 4]) +>>> print(r_squared) +1.0 +``` + +## Limitations and Bias +R^2 is a statistical measure of the goodness of fit of a regression model. It represents the proportion of the variance in the dependent variable that is predictable from the independent variables. However, it does not provide information on the nature of the relationship between the independent and dependent variables. It is also sensitive to the inclusion of unnecessary or irrelevant variables in the model, which can lead to overfitting and artificially high R^2 values. + +## Citation + +```bibtex +@article{r_squared_model, + title={The R^2 Model Metric: A Comprehensive Guide}, + author={John Doe}, + journal={Journal of Model Evaluation}, + volume={10}, + number={2}, + pages={101-112}, + year={2022}, + publisher={Model Evaluation Society}} +``` + +## Further References + +- [The Open University: R-Squared](https://www.open.edu/openlearn/ocw/mod/oucontent/view.php?id=55450§ion=3.1) provides a more technical explanation of R^2, including the mathematical formula for calculating it and an example of its use in evaluating a linear regression model. + +- [Khan Academy: R-Squared](https://www.khanacademy.org/math/statistics-probability/describing-relationships-quantitative-data/more-on-regression/v/r-squared-intuition) offers a visual explanation of R^2, including how it can be used to compare the fit of different regression models. diff --git a/ST/evaluate/metrics/r_squared/app.py b/ST/evaluate/metrics/r_squared/app.py new file mode 100644 index 0000000000000000000000000000000000000000..c03a6103bd583453f86e8e53bb84f8784b754ecb --- /dev/null +++ b/ST/evaluate/metrics/r_squared/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("r_squared") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/r_squared/r_squared.py b/ST/evaluate/metrics/r_squared/r_squared.py new file mode 100644 index 0000000000000000000000000000000000000000..75e0f7e00fca51bff42f924b4d98042d0848f864 --- /dev/null +++ b/ST/evaluate/metrics/r_squared/r_squared.py @@ -0,0 +1,115 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""R squared metric.""" + + +import datasets +import numpy as np + +import evaluate + + +_CITATION = """ +@article{williams2006relationship, +title={The relationship between R2 and the correlation coefficient}, +author={Williams, James}, +journal={Journal of Statistics Education}, +volume={14}, +number={2}, +year={2006} +} +""" + +_DESCRIPTION = """ +R^2 (R Squared) is a statistical measure of the goodness of fit of a regression model. It represents the proportion of the variance in the dependent variable that is predictable from the independent variables. + +The R^2 value ranges from 0 to 1, with a higher value indicating a better fit. A value of 0 means that the model does not explain any of the variance in the dependent variable, while a value of 1 means that the model explains all of the variance. + +R^2 can be calculated using the following formula: + +r_squared = 1 - (Sum of Squared Errors / Sum of Squared Total) + +where the Sum of Squared Errors is the sum of the squared differences between the predicted values and the true values, and the Sum of Squared Total is the sum of the squared differences between the true values and the mean of the true values. +""" + +_KWARGS_DESCRIPTION = """ +Computes the R Squared metric. + +Args: + predictions: List of predicted values of the dependent variable + references: List of true values of the dependent variable + zero_division: Which value to substitute as a metric value when encountering zero division. Should be one of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + R^2 value ranging from 0 to 1, with a higher value indicating a better fit. + +Examples: + >>> r2_metric = evaluate.load("r_squared") + >>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[0.9, 2.1, 3.2, 3.8]) + >>> print(r_squared) + 0.98 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class r_squared(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float", id="sequence"), + "references": datasets.Value("float", id="sequence"), + } + ), + codebase_urls=["https://github.com/scikit-learn/scikit-learn/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Coefficient_of_determination", + ], + ) + + def _compute(self, predictions=None, references=None): + """ + Computes the coefficient of determination (R-squared) of predictions with respect to references. + + Parameters: + predictions (List or np.ndarray): The predicted values. + references (List or np.ndarray): The true/reference values. + + Returns: + float: The R-squared value, rounded to 3 decimal places. + """ + predictions = np.array(predictions) + references = np.array(references) + + # Calculate mean of the references + mean_references = np.mean(references) + + # Calculate sum of squared residuals + ssr = np.sum((predictions - references) ** 2) + + # Calculate sum of squared total + sst = np.sum((references - mean_references) ** 2) + + # Calculate R Squared + r_squared = 1 - (ssr / sst) + + # Round off to 3 decimal places + rounded_r_squared = round(r_squared, 3) + + return rounded_r_squared diff --git a/ST/evaluate/metrics/r_squared/requirements.txt b/ST/evaluate/metrics/r_squared/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b77fbc416b18f5668d2f466f2bb0f12957bdde9 --- /dev/null +++ b/ST/evaluate/metrics/r_squared/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} diff --git a/ST/evaluate/metrics/recall/README.md b/ST/evaluate/metrics/recall/README.md new file mode 100644 index 0000000000000000000000000000000000000000..58b021c0709484732e0e0524735e7a6a51e44f3f --- /dev/null +++ b/ST/evaluate/metrics/recall/README.md @@ -0,0 +1,132 @@ +--- +title: Recall +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: + Recall = TP / (TP + FN) + Where TP is the true positives and FN is the false negatives. +--- + +# Metric Card for Recall + + +## Metric Description + +Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: +Recall = TP / (TP + FN) +Where TP is the number of true positives and FN is the number of false negatives. + + +## How to Use + +At minimum, this metric takes as input two `list`s, each containing `int`s: predictions and references. + +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +["{'recall': 1.0}"] +``` + + +### Inputs +- **predictions** (`list` of `int`): The predicted labels. +- **references** (`list` of `int`): The ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. + - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. + - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. + - `0`: If there is a zero division, the return value is `0`. + - `1`: If there is a zero division, the return value is `1`. + + +### Output Values +- **recall**(`float`, or `array` of `float`, for multiclass targets): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. + +Output Example(s): +```python +{'recall': 1.0} +``` +```python +{'recall': array([1., 0., 0.])} +``` + +This metric outputs a dictionary with one entry, `'recall'`. + + +#### Values from Popular Papers + + +### Examples + +Example 1-A simple example with some errors +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) +>>> print(results) +{'recall': 0.6666666666666666} +``` + +Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) +>>> print(results) +{'recall': 0.5} +``` + +Example 3-The same example as Example 1, but with `sample_weight` included. +```python +>>> recall_metric = evaluate.load('recall') +>>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) +>>> print(results) +{'recall': 0.55} +``` + +Example 4-A multiclass example, using different averages. +```python +>>> recall_metric = evaluate.load('recall') +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average=None) +>>> print(results) +{'recall': array([1., 0., 0.])} +``` + + +## Limitations and Bias + + +## Citation(s) +```bibtex +@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} +``` + + +## Further References diff --git a/ST/evaluate/metrics/recall/app.py b/ST/evaluate/metrics/recall/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3b273568b99eb1c2ddfe2c685bb2ec85afbe1b56 --- /dev/null +++ b/ST/evaluate/metrics/recall/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("recall") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/recall/recall.py b/ST/evaluate/metrics/recall/recall.py new file mode 100644 index 0000000000000000000000000000000000000000..8522cfcf66563befa3b8fa3ba5be1f1145fadb7e --- /dev/null +++ b/ST/evaluate/metrics/recall/recall.py @@ -0,0 +1,135 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Recall metric.""" + +import datasets +from sklearn.metrics import recall_score + +import evaluate + + +_DESCRIPTION = """ +Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: +Recall = TP / (TP + FN) +Where TP is the true positives and FN is the false negatives. +""" + + +_KWARGS_DESCRIPTION = """ +Args: +- **predictions** (`list` of `int`): The predicted labels. +- **references** (`list` of `int`): The ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. + - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. + - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. + - `0`: If there is a zero division, the return value is `0`. + - `1`: If there is a zero division, the return value is `1`. + +Returns: +- **recall** (`float`, or `array` of `float`): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. + +Examples: + + Example 1-A simple example with some errors + >>> recall_metric = evaluate.load('recall') + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) + >>> print(results) + {'recall': 0.6666666666666666} + + Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. + >>> recall_metric = evaluate.load('recall') + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) + >>> print(results) + {'recall': 0.5} + + Example 3-The same example as Example 1, but with `sample_weight` included. + >>> recall_metric = evaluate.load('recall') + >>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) + >>> print(results) + {'recall': 0.55} + + Example 4-A multiclass example, using different averages. + >>> recall_metric = evaluate.load('recall') + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average=None) + >>> print(results) + {'recall': array([1., 0., 0.])} +""" + + +_CITATION = """ +@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Recall(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html"], + ) + + def _compute( + self, + predictions, + references, + labels=None, + pos_label=1, + average="binary", + sample_weight=None, + zero_division="warn", + ): + score = recall_score( + references, + predictions, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division, + ) + return {"recall": float(score) if score.size == 1 else score} diff --git a/ST/evaluate/metrics/recall/requirements.txt b/ST/evaluate/metrics/recall/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/recall/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/rl_reliability/README.md b/ST/evaluate/metrics/rl_reliability/README.md new file mode 100644 index 0000000000000000000000000000000000000000..372302c049278bc49ee848e45a61bc71e4ad17b6 --- /dev/null +++ b/ST/evaluate/metrics/rl_reliability/README.md @@ -0,0 +1,118 @@ +--- +title: RL Reliability +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +--- + +# Metric Card for RL Reliability + +## Metric Description +The RL Reliability Metrics library provides a set of metrics for measuring the reliability of reinforcement learning (RL) algorithms. + +## How to Use + +```python +import evaluate +import numpy as np + +rl_reliability = evaluate.load("rl_reliability", "online") +results = rl_reliability.compute( + timesteps=[np.linspace(0, 2000000, 1000)], + rewards=[np.linspace(0, 100, 1000)] + ) + +rl_reliability = evaluate.load("rl_reliability", "offline") +results = rl_reliability.compute( + timesteps=[np.linspace(0, 2000000, 1000)], + rewards=[np.linspace(0, 100, 1000)] + ) +``` + + +### Inputs +- **timesteps** *(List[int]): For each run a an list/array with its timesteps.* +- **rewards** *(List[float]): For each run a an list/array with its rewards.* + +KWARGS: +- **baseline="default"** *(Union[str, float]) Normalization used for curves. When `"default"` is passed the curves are normalized by their range in the online setting and by the median performance across runs in the offline case. When a float is passed the curves are divided by that value.* +- **eval_points=[50000, 150000, ..., 2000000]** *(List[int]) Statistics will be computed at these points* +- **freq_thresh=0.01** *(float) Frequency threshold for low-pass filtering.* +- **window_size=100000** *(int) Defines a window centered at each eval point.* +- **window_size_trimmed=99000** *(int) To handle shortened curves due to differencing* +- **alpha=0.05** *(float)The "value at risk" (VaR) cutoff point, a float in the range [0,1].* + +### Output Values + +In `"online"` mode: +- HighFreqEnergyWithinRuns: High Frequency across Time (DT) +- IqrWithinRuns: IQR across Time (DT) +- MadWithinRuns: 'MAD across Time (DT) +- StddevWithinRuns: Stddev across Time (DT) +- LowerCVaROnDiffs: Lower CVaR on Differences (SRT) +- UpperCVaROnDiffs: Upper CVaR on Differences (SRT) +- MaxDrawdown: Max Drawdown (LRT) +- LowerCVaROnDrawdown: Lower CVaR on Drawdown (LRT) +- UpperCVaROnDrawdown: Upper CVaR on Drawdown (LRT) +- LowerCVaROnRaw: Lower CVaR on Raw +- UpperCVaROnRaw: Upper CVaR on Raw +- IqrAcrossRuns: IQR across Runs (DR) +- MadAcrossRuns: MAD across Runs (DR) +- StddevAcrossRuns: Stddev across Runs (DR) +- LowerCVaROnAcross: Lower CVaR across Runs (RR) +- UpperCVaROnAcross: Upper CVaR across Runs (RR) +- MedianPerfDuringTraining: Median Performance across Runs + +In `"offline"` mode: +- MadAcrossRollouts: MAD across rollouts (DF) +- IqrAcrossRollouts: IQR across rollouts (DF) +- LowerCVaRAcrossRollouts: Lower CVaR across rollouts (RF) +- UpperCVaRAcrossRollouts: Upper CVaR across rollouts (RF) +- MedianPerfAcrossRollouts: Median Performance across rollouts + + +### Examples +First get the sample data from the repository: + +```bash +wget https://storage.googleapis.com/rl-reliability-metrics/data/tf_agents_example_csv_dataset.tgz +tar -xvzf tf_agents_example_csv_dataset.tgz +``` + +Load the sample data: +```python +dfs = [pd.read_csv(f"./csv_data/sac_humanoid_{i}_train.csv") for i in range(1, 4)] +``` + +Compute the metrics: +```python +rl_reliability = evaluate.load("rl_reliability", "online") +rl_reliability.compute(timesteps=[df["Metrics/EnvironmentSteps"] for df in dfs], + rewards=[df["Metrics/AverageReturn"] for df in dfs]) +``` + +## Limitations and Bias +This implementation of RL reliability metrics does not compute permutation tests to determine whether algorithms are statistically different in their metric values and also does not compute bootstrap confidence intervals on the rankings of the algorithms. See the [original library](https://github.com/google-research/rl-reliability-metrics/) for more resources. + +## Citation + +```bibtex +@conference{rl_reliability_metrics, + title = {Measuring the Reliability of Reinforcement Learning Algorithms}, + author = {Stephanie CY Chan, Sam Fishman, John Canny, Anoop Korattikara, and Sergio Guadarrama}, + booktitle = {International Conference on Learning Representations, Addis Ababa, Ethiopia}, + year = 2020, +} +``` + +## Further References +- Homepage: https://github.com/google-research/rl-reliability-metrics diff --git a/ST/evaluate/metrics/rl_reliability/app.py b/ST/evaluate/metrics/rl_reliability/app.py new file mode 100644 index 0000000000000000000000000000000000000000..58f118ded69c2b589eb1ab89e9c5fd93ab7eb41d --- /dev/null +++ b/ST/evaluate/metrics/rl_reliability/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("rl_reliability", "online") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/rl_reliability/requirements.txt b/ST/evaluate/metrics/rl_reliability/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..514080bf1a2b571122069de43e6faefc6a4d5d98 --- /dev/null +++ b/ST/evaluate/metrics/rl_reliability/requirements.txt @@ -0,0 +1,5 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/google-research/rl-reliability-metrics +scipy +tensorflow +gin-config \ No newline at end of file diff --git a/ST/evaluate/metrics/rl_reliability/rl_reliability.py b/ST/evaluate/metrics/rl_reliability/rl_reliability.py new file mode 100644 index 0000000000000000000000000000000000000000..34a9c4570cbc2fcd7f4392886b32de6fa17e4dfd --- /dev/null +++ b/ST/evaluate/metrics/rl_reliability/rl_reliability.py @@ -0,0 +1,186 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Computes the RL Reliability Metrics.""" + +import datasets +import numpy as np +from rl_reliability_metrics.evaluation import eval_metrics +from rl_reliability_metrics.metrics import metrics_offline, metrics_online + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +DEFAULT_EVAL_POINTS = [ + 50000, + 150000, + 250000, + 350000, + 450000, + 550000, + 650000, + 750000, + 850000, + 950000, + 1050000, + 1150000, + 1250000, + 1350000, + 1450000, + 1550000, + 1650000, + 1750000, + 1850000, + 1950000, +] + +N_RUNS_RECOMMENDED = 10 + +_CITATION = """\ +@conference{rl_reliability_metrics, + title = {Measuring the Reliability of Reinforcement Learning Algorithms}, + author = {Stephanie CY Chan, Sam Fishman, John Canny, Anoop Korattikara, and Sergio Guadarrama}, + booktitle = {International Conference on Learning Representations, Addis Ababa, Ethiopia}, + year = 2020, +} +""" + +_DESCRIPTION = """\ +Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +""" + + +_KWARGS_DESCRIPTION = """ +Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +Args: + timestamps: list of timestep lists/arrays that serve as index. + rewards: list of reward lists/arrays of each experiment. +Returns: + dictionary: a set of reliability metrics +Examples: + >>> import numpy as np + >>> rl_reliability = evaluate.load("rl_reliability", "online") + >>> results = rl_reliability.compute( + ... timesteps=[np.linspace(0, 2000000, 1000)], + ... rewards=[np.linspace(0, 100, 1000)] + ... ) + >>> print(results["LowerCVaROnRaw"].round(4)) + [0.0258] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class RLReliability(evaluate.Metric): + """Computes the RL Reliability Metrics.""" + + def _info(self): + if self.config_name not in ["online", "offline"]: + raise KeyError("""You should supply a configuration name selected in '["online", "offline"]'""") + + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "timesteps": datasets.Sequence(datasets.Value("int64")), + "rewards": datasets.Sequence(datasets.Value("float")), + } + ), + homepage="https://github.com/google-research/rl-reliability-metrics", + ) + + def _compute( + self, + timesteps, + rewards, + baseline="default", + freq_thresh=0.01, + window_size=100000, + window_size_trimmed=99000, + alpha=0.05, + eval_points=None, + ): + if len(timesteps) < N_RUNS_RECOMMENDED: + logger.warning( + f"For robust statistics it is recommended to use at least {N_RUNS_RECOMMENDED} runs whereas you provided {len(timesteps)}." + ) + + curves = [] + for timestep, reward in zip(timesteps, rewards): + curves.append(np.stack([timestep, reward])) + + if self.config_name == "online": + if baseline == "default": + baseline = "curve_range" + if eval_points is None: + eval_points = DEFAULT_EVAL_POINTS + + metrics = [ + metrics_online.HighFreqEnergyWithinRuns(thresh=freq_thresh), + metrics_online.IqrWithinRuns( + window_size=window_size_trimmed, eval_points=eval_points, baseline=baseline + ), + metrics_online.IqrAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.LowerCVaROnDiffs(baseline=baseline), + metrics_online.LowerCVaROnDrawdown(baseline=baseline), + metrics_online.LowerCVaROnAcross( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.LowerCVaROnRaw(alpha=alpha, baseline=baseline), + metrics_online.MadAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.MadWithinRuns( + eval_points=eval_points, window_size=window_size_trimmed, baseline=baseline + ), + metrics_online.MaxDrawdown(), + metrics_online.StddevAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.StddevWithinRuns( + eval_points=eval_points, window_size=window_size_trimmed, baseline=baseline + ), + metrics_online.UpperCVaROnAcross( + alpha=alpha, + lowpass_thresh=freq_thresh, + eval_points=eval_points, + window_size=window_size, + baseline=baseline, + ), + metrics_online.UpperCVaROnDiffs(alpha=alpha, baseline=baseline), + metrics_online.UpperCVaROnDrawdown(alpha=alpha, baseline=baseline), + metrics_online.UpperCVaROnRaw(alpha=alpha, baseline=baseline), + metrics_online.MedianPerfDuringTraining(window_size=window_size, eval_points=eval_points), + ] + else: + if baseline == "default": + baseline = "median_perf" + + metrics = [ + metrics_offline.MadAcrossRollouts(baseline=baseline), + metrics_offline.IqrAcrossRollouts(baseline=baseline), + metrics_offline.StddevAcrossRollouts(baseline=baseline), + metrics_offline.LowerCVaRAcrossRollouts(alpha=alpha, baseline=baseline), + metrics_offline.UpperCVaRAcrossRollouts(alpha=alpha, baseline=baseline), + metrics_offline.MedianPerfAcrossRollouts(baseline=None), + ] + + evaluator = eval_metrics.Evaluator(metrics=metrics) + result = evaluator.compute_metrics(curves) + return result diff --git a/ST/evaluate/metrics/roc_auc/README.md b/ST/evaluate/metrics/roc_auc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99e3314c958caed6d05fbaa3f8444e734904780c --- /dev/null +++ b/ST/evaluate/metrics/roc_auc/README.md @@ -0,0 +1,204 @@ +--- +title: ROC AUC +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + + This metric has three separate use cases: + - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. + - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label. + - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label. +--- + +# Metric Card for ROC AUC + + +## Metric Description +This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + +This metric has three separate use cases: +- **binary**: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. +- **multiclass**: The case in which there can be more than two different label classes, but each example still gets only one label. +- **multilabel**: The case in which there can be more than two different label classes, and each example can have more than one label. + + +## How to Use +At minimum, this metric requires references and prediction scores: +```python +>>> roc_auc_score = evaluate.load("roc_auc") +>>> refs = [1, 0, 1, 1, 0, 0] +>>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] +>>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) +>>> print(round(results['roc_auc'], 2)) +0.78 +``` + +The default implementation of this metric is the **binary** implementation. If employing the **multiclass** or **multilabel** use cases, the keyword `"multiclass"` or `"multilabel"` must be specified when loading the metric: +- In the **multiclass** case, the metric is loaded with: + ```python + >>> roc_auc_score = evaluate.load("roc_auc", "multiclass") + ``` +- In the **multilabel** case, the metric is loaded with: + ```python + >>> roc_auc_score = evaluate.load("roc_auc", "multilabel") + ``` + +See the [Examples Section Below](#examples_section) for more extensive examples. + + +### Inputs +- **`references`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples,) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- **`prediction_scores`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples, n_classes). The probability estimates must sum to 1 across the possible classes. + - multilabel: expects an array-like of shape (n_samples, n_classes) +- **`average`** (`str`): Type of average, and is ignored in the binary use case. Defaults to `'macro'`. Options are: + - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). + - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. + - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. +- **`sample_weight`** (array-like of shape (n_samples,)): Sample weights. Defaults to None. +- **`max_fpr`** (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. +- **`multi_class`** (`str`): Only used for multiclass targets, in which case it is required. Determines the type of configuration to use. Options are: + - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. + - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. +- **`labels`** (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in `prediction_scores` is used. Defaults to `None`. + +### Output Values +This metric returns a dict containing the `roc_auc` score. The score is a `float`, unless it is the multilabel case with `average=None`, in which case the score is a numpy `array` with entries of type `float`. + +The output therefore generally takes the following format: +```python +{'roc_auc': 0.778} +``` + +In contrast, though, the output takes the following format in the multilabel case when `average=None`: +```python +{'roc_auc': array([0.83333333, 0.375, 0.94444444])} +``` + +ROC AUC scores can take on any value between `0` and `1`, inclusive. + +#### Values from Popular Papers + + +### Examples +Example 1, the **binary** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc") +>>> refs = [1, 0, 1, 1, 0, 0] +>>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] +>>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) +>>> print(round(results['roc_auc'], 2)) +0.78 +``` + +Example 2, the **multiclass** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc", "multiclass") +>>> refs = [1, 0, 1, 2, 2, 0] +>>> pred_scores = [[0.3, 0.5, 0.2], +... [0.7, 0.2, 0.1], +... [0.005, 0.99, 0.005], +... [0.2, 0.3, 0.5], +... [0.1, 0.1, 0.8], +... [0.1, 0.7, 0.2]] +>>> results = roc_auc_score.compute(references=refs, +... prediction_scores=pred_scores, +... multi_class='ovr') +>>> print(round(results['roc_auc'], 2)) +0.85 +``` + +Example 3, the **multilabel** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc", "multilabel") +>>> refs = [[1, 1, 0], +... [1, 1, 0], +... [0, 1, 0], +... [0, 0, 1], +... [0, 1, 1], +... [1, 0, 1]] +>>> pred_scores = [[0.3, 0.5, 0.2], +... [0.7, 0.2, 0.1], +... [0.005, 0.99, 0.005], +... [0.2, 0.3, 0.5], +... [0.1, 0.1, 0.8], +... [0.1, 0.7, 0.2]] +>>> results = roc_auc_score.compute(references=refs, +... prediction_scores=pred_scores, +... average=None) +>>> print([round(res, 2) for res in results['roc_auc']) +[0.83, 0.38, 0.94] +``` + + +## Limitations and Bias + + +## Citation +```bibtex +@article{doi:10.1177/0272989X8900900307, +author = {Donna Katzman McClish}, +title ={Analyzing a Portion of the ROC Curve}, +journal = {Medical Decision Making}, +volume = {9}, +number = {3}, +pages = {190-195}, +year = {1989}, +doi = {10.1177/0272989X8900900307}, + note ={PMID: 2668680}, +URL = {https://doi.org/10.1177/0272989X8900900307}, +eprint = {https://doi.org/10.1177/0272989X8900900307} +} +``` + +```bibtex +@article{10.1023/A:1010920819831, +author = {Hand, David J. and Till, Robert J.}, +title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, +year = {2001}, +issue_date = {November 2001}, +publisher = {Kluwer Academic Publishers}, +address = {USA}, +volume = {45}, +number = {2}, +issn = {0885-6125}, +url = {https://doi.org/10.1023/A:1010920819831}, +doi = {10.1023/A:1010920819831}, +journal = {Mach. Learn.}, +month = {oct}, +pages = {171–186}, +numpages = {16}, +keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} +} +``` + +```bibtex +@article{scikit-learn, +title={Scikit-learn: Machine Learning in {P}ython}, +author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, +journal={Journal of Machine Learning Research}, +volume={12}, +pages={2825--2830}, +year={2011} +} +``` + +## Further References +This implementation is a wrapper around the [Scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html). Much of the documentation here was adapted from their existing documentation, as well. + +The [Guide to ROC and AUC](https://youtu.be/iCZJfO-7C5Q) video from the channel Data Science Bits is also very informative. diff --git a/ST/evaluate/metrics/roc_auc/app.py b/ST/evaluate/metrics/roc_auc/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4830128cf25675a2a6fe810c36f0469ff40f6c67 --- /dev/null +++ b/ST/evaluate/metrics/roc_auc/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("roc_auc") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/roc_auc/requirements.txt b/ST/evaluate/metrics/roc_auc/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/roc_auc/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/roc_auc/roc_auc.py b/ST/evaluate/metrics/roc_auc/roc_auc.py new file mode 100644 index 0000000000000000000000000000000000000000..604f93ab3d7b5c0cd6428df1d67a3a1791a954e3 --- /dev/null +++ b/ST/evaluate/metrics/roc_auc/roc_auc.py @@ -0,0 +1,191 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric.""" + +import datasets +from sklearn.metrics import roc_auc_score + +import evaluate + + +_DESCRIPTION = """ +This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + +This metric has three separate use cases: + - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. + - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label. + - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label. +""" + +_KWARGS_DESCRIPTION = """ +Args: +- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples,) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples, n_classes) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- average (`str`): Type of average, and is ignored in the binary use case. Defaults to 'macro'. Options are: + - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). + - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. + - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. +- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None. +- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. +- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are: + - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. + - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. +- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in + `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in + `prediction_scores` is used. Defaults to `None`. +Returns: + roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average='None'`. Otherwise, returns `float`. +Examples: + Example 1: + >>> roc_auc_score = evaluate.load("roc_auc") + >>> refs = [1, 0, 1, 1, 0, 0] + >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) + >>> print(round(results['roc_auc'], 2)) + 0.78 + + Example 2: + >>> roc_auc_score = evaluate.load("roc_auc", "multiclass") + >>> refs = [1, 0, 1, 2, 2, 0] + >>> pred_scores = [[0.3, 0.5, 0.2], + ... [0.7, 0.2, 0.1], + ... [0.005, 0.99, 0.005], + ... [0.2, 0.3, 0.5], + ... [0.1, 0.1, 0.8], + ... [0.1, 0.7, 0.2]] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class='ovr') + >>> print(round(results['roc_auc'], 2)) + 0.85 + + Example 3: + >>> roc_auc_score = evaluate.load("roc_auc", "multilabel") + >>> refs = [[1, 1, 0], + ... [1, 1, 0], + ... [0, 1, 0], + ... [0, 0, 1], + ... [0, 1, 1], + ... [1, 0, 1]] + >>> pred_scores = [[0.3, 0.5, 0.2], + ... [0.7, 0.2, 0.1], + ... [0.005, 0.99, 0.005], + ... [0.2, 0.3, 0.5], + ... [0.1, 0.1, 0.8], + ... [0.1, 0.7, 0.2]] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None) + >>> print([round(res, 2) for res in results['roc_auc']]) + [0.83, 0.38, 0.94] +""" + +_CITATION = """\ +@article{doi:10.1177/0272989X8900900307, +author = {Donna Katzman McClish}, +title ={Analyzing a Portion of the ROC Curve}, +journal = {Medical Decision Making}, +volume = {9}, +number = {3}, +pages = {190-195}, +year = {1989}, +doi = {10.1177/0272989X8900900307}, + note ={PMID: 2668680}, +URL = {https://doi.org/10.1177/0272989X8900900307}, +eprint = {https://doi.org/10.1177/0272989X8900900307} +} + + +@article{10.1023/A:1010920819831, +author = {Hand, David J. and Till, Robert J.}, +title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, +year = {2001}, +issue_date = {November 2001}, +publisher = {Kluwer Academic Publishers}, +address = {USA}, +volume = {45}, +number = {2}, +issn = {0885-6125}, +url = {https://doi.org/10.1023/A:1010920819831}, +doi = {10.1023/A:1010920819831}, +journal = {Mach. Learn.}, +month = {oct}, +pages = {171–186}, +numpages = {16}, +keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} +} + + +@article{scikit-learn, +title={Scikit-learn: Machine Learning in {P}ython}, +author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, +journal={Journal of Machine Learning Research}, +volume={12}, +pages={2825--2830}, +year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ROCAUC(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "prediction_scores": datasets.Sequence(datasets.Value("float")), + "references": datasets.Value("int32"), + } + if self.config_name == "multiclass" + else { + "references": datasets.Sequence(datasets.Value("int32")), + "prediction_scores": datasets.Sequence(datasets.Value("float")), + } + if self.config_name == "multilabel" + else { + "references": datasets.Value("int32"), + "prediction_scores": datasets.Value("float"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html"], + ) + + def _compute( + self, + references, + prediction_scores, + average="macro", + sample_weight=None, + max_fpr=None, + multi_class="raise", + labels=None, + ): + return { + "roc_auc": roc_auc_score( + references, + prediction_scores, + average=average, + sample_weight=sample_weight, + max_fpr=max_fpr, + multi_class=multi_class, + labels=labels, + ) + } diff --git a/ST/evaluate/metrics/rouge/README.md b/ST/evaluate/metrics/rouge/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5c143c4978d02231681009e82f9b1154c0733414 --- /dev/null +++ b/ST/evaluate/metrics/rouge/README.md @@ -0,0 +1,161 @@ +--- +title: ROUGE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for + evaluating automatic summarization and machine translation software in natural language processing. + The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + + Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + + This metrics is a wrapper around Google Research reimplementation of ROUGE: + https://github.com/google-research/google-research/tree/master/rouge +--- + +# Metric Card for ROUGE + +## Metric Description +ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + +Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + +This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) + +## How to Use +At minimum, this metric takes as input a list of predictions and a list of references: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references) +>>> print(results) +{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +``` + +One can also pass a custom tokenizer which is especially useful for non-latin languages. +```python +>>> results = rouge.compute(predictions=predictions, +... references=references, + tokenizer=lambda x: x.split()) +>>> print(results) +{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +``` + +It can also deal with lists of references for each predictions: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello there", "general kenobi"] +>>> references = [["hello", "there"], ["general kenobi", "general yoda"]] +>>> results = rouge.compute(predictions=predictions, +... references=references) +>>> print(results) +{'rouge1': 0.8333, 'rouge2': 0.5, 'rougeL': 0.8333, 'rougeLsum': 0.8333}``` +``` + +### Inputs +- **predictions** (`list`): list of predictions to score. Each prediction + should be a string with tokens separated by spaces. +- **references** (`list` or `list[list]`): list of reference for each prediction or a list of several references per prediction. Each + reference should be a string with tokens separated by spaces. +- **rouge_types** (`list`): A list of rouge types to calculate. Defaults to `['rouge1', 'rouge2', 'rougeL', 'rougeLsum']`. + - Valid rouge types: + - `"rouge1"`: unigram (1-gram) based scoring + - `"rouge2"`: bigram (2-gram) based scoring + - `"rougeL"`: Longest common subsequence based scoring. + - `"rougeLSum"`: splits text using `"\n"` + - See [here](https://github.com/huggingface/datasets/issues/617) for more information +- **use_aggregator** (`boolean`): If True, returns aggregates. Defaults to `True`. +- **use_stemmer** (`boolean`): If `True`, uses Porter stemmer to strip word suffixes. Defaults to `False`. + +### Output Values +The output is a dictionary with one entry for each rouge type in the input list `rouge_types`. If `use_aggregator=False`, each dictionary entry is a list of scores, with one score for each sentence. E.g. if `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=False`, the output is: + +```python +{'rouge1': [0.6666666666666666, 1.0], 'rouge2': [0.0, 1.0]} +``` + +If `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=True`, the output is of the following format: +```python +{'rouge1': 1.0, 'rouge2': 1.0} +``` + +The ROUGE values are in the range of 0 to 1. + + +#### Values from Popular Papers + + +### Examples +An example without aggregation: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... use_aggregator=False) +>>> print(list(results.keys())) +['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] +>>> print(results["rouge1"]) +[0.5, 0.0] +``` + +The same example, but with aggregation: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... use_aggregator=True) +>>> print(list(results.keys())) +['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] +>>> print(results["rouge1"]) +0.25 +``` + +The same example, but only calculating `rouge_1`: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... rouge_types=['rouge_1'], +... use_aggregator=True) +>>> print(list(results.keys())) +['rouge1'] +>>> print(results["rouge1"]) +0.25 +``` + +## Limitations and Bias +See [Schluter (2017)](https://aclanthology.org/E17-2007/) for an in-depth discussion of many of ROUGE's limits. + +## Citation +```bibtex +@inproceedings{lin-2004-rouge, + title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", + author = "Lin, Chin-Yew", + booktitle = "Text Summarization Branches Out", + month = jul, + year = "2004", + address = "Barcelona, Spain", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W04-1013", + pages = "74--81", +} +``` + +## Further References +- This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) \ No newline at end of file diff --git a/ST/evaluate/metrics/rouge/app.py b/ST/evaluate/metrics/rouge/app.py new file mode 100644 index 0000000000000000000000000000000000000000..38eb3fd38c21e1715cab970448654c9dc8a4f7bf --- /dev/null +++ b/ST/evaluate/metrics/rouge/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("rouge") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/rouge/requirements.txt b/ST/evaluate/metrics/rouge/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7df8b26f3b8f204880d345de2e27b1349483a06a --- /dev/null +++ b/ST/evaluate/metrics/rouge/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +absl-py +nltk +rouge_score>=0.1.2 \ No newline at end of file diff --git a/ST/evaluate/metrics/rouge/rouge.py b/ST/evaluate/metrics/rouge/rouge.py new file mode 100644 index 0000000000000000000000000000000000000000..353301cca11fb5c7d4f0b0e70cde1560b4139bc7 --- /dev/null +++ b/ST/evaluate/metrics/rouge/rouge.py @@ -0,0 +1,158 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ROUGE metric from Google Research github repo. """ + +# The dependencies in https://github.com/google-research/google-research/blob/master/rouge/requirements.txt +import absl # Here to have a nice missing dependency error message early on +import datasets +import nltk # Here to have a nice missing dependency error message early on +import numpy # Here to have a nice missing dependency error message early on +import six # Here to have a nice missing dependency error message early on +from rouge_score import rouge_scorer, scoring + +import evaluate + + +_CITATION = """\ +@inproceedings{lin-2004-rouge, + title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", + author = "Lin, Chin-Yew", + booktitle = "Text Summarization Branches Out", + month = jul, + year = "2004", + address = "Barcelona, Spain", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W04-1013", + pages = "74--81", +} +""" + +_DESCRIPTION = """\ +ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for +evaluating automatic summarization and machine translation software in natural language processing. +The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + +Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + +This metrics is a wrapper around Google Research reimplementation of ROUGE: +https://github.com/google-research/google-research/tree/master/rouge +""" + +_KWARGS_DESCRIPTION = """ +Calculates average rouge scores for a list of hypotheses and references +Args: + predictions: list of predictions to score. Each prediction + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. + rouge_types: A list of rouge types to calculate. + Valid names: + `"rouge{n}"` (e.g. `"rouge1"`, `"rouge2"`) where: {n} is the n-gram based scoring, + `"rougeL"`: Longest common subsequence based scoring. + `"rougeLsum"`: rougeLsum splits text using `"\n"`. + See details in https://github.com/huggingface/datasets/issues/617 + use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes. + use_aggregator: Return aggregates if this is set to True +Returns: + rouge1: rouge_1 (f1), + rouge2: rouge_2 (f1), + rougeL: rouge_l (f1), + rougeLsum: rouge_lsum (f1) +Examples: + + >>> rouge = evaluate.load('rouge') + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> results = rouge.compute(predictions=predictions, references=references) + >>> print(results) + {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +""" + + +class Tokenizer: + """Helper class to wrap a callable into a class with a `tokenize` method as used by rouge-score.""" + + def __init__(self, tokenizer_func): + self.tokenizer_func = tokenizer_func + + def tokenize(self, text): + return self.tokenizer_func(text) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Rouge(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence")), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"], + reference_urls=[ + "https://en.wikipedia.org/wiki/ROUGE_(metric)", + "https://github.com/google-research/google-research/tree/master/rouge", + ], + ) + + def _compute( + self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False, tokenizer=None + ): + if rouge_types is None: + rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"] + + multi_ref = isinstance(references[0], list) + + if tokenizer is not None: + tokenizer = Tokenizer(tokenizer) + + scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer, tokenizer=tokenizer) + if use_aggregator: + aggregator = scoring.BootstrapAggregator() + else: + scores = [] + + for ref, pred in zip(references, predictions): + if multi_ref: + score = scorer.score_multi(ref, pred) + else: + score = scorer.score(ref, pred) + if use_aggregator: + aggregator.add_scores(score) + else: + scores.append(score) + + if use_aggregator: + result = aggregator.aggregate() + for key in result: + result[key] = result[key].mid.fmeasure + + else: + result = {} + for key in scores[0]: + result[key] = list(score[key].fmeasure for score in scores) + + return result diff --git a/ST/evaluate/metrics/sacrebleu/README.md b/ST/evaluate/metrics/sacrebleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a86a070022c235a2e9297597c4301b95045689a0 --- /dev/null +++ b/ST/evaluate/metrics/sacrebleu/README.md @@ -0,0 +1,119 @@ +--- +title: SacreBLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. + Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text. + It also knows all the standard test sets and handles downloading, processing, and tokenization for you. + + See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. +--- + +# Metric Card for SacreBLEU + + +## Metric Description +SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official Workshop on Machine Translation (WMT) scores but works with plain text. It also knows all the standard test sets and handles downloading, processing, and tokenization. + +See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. + +## How to Use +This metric takes a set of predictions and a set of references as input, along with various optional parameters. + + +```python +>>> predictions = ["hello there general kenobi", "foo bar foobar"] +>>> references = [["hello there general kenobi", "hello there !"], +... ["foo bar foobar", "foo bar foobar"]] +>>> sacrebleu = evaluate.load("sacrebleu") +>>> results = sacrebleu.compute(predictions=predictions, +... references=references) +>>> print(list(results.keys())) +['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] +>>> print(round(results["score"], 1)) +100.0 +``` + +### Inputs +- **`predictions`** (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. +- **`references`** (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). +- **`smooth_method`** (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: + - `'none'`: no smoothing + - `'floor'`: increment zero counts + - `'add-k'`: increment num/denom by k for n>1 + - `'exp'`: exponential decay +- **`smooth_value`** (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). +- **`tokenize`** (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: + - `'none'`: No tokenization. + - `'zh'`: Chinese tokenization. + - `'13a'`: mimics the `mteval-v13a` script from Moses. + - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses + - `'char'`: Language-agnostic character-level tokenization. + - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). +- **`lowercase`** (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. +- **`force`** (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. +- **`use_effective_order`** (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. + +### Output Values +- `score`: BLEU score +- `counts`: Counts +- `totals`: Totals +- `precisions`: Precisions +- `bp`: Brevity penalty +- `sys_len`: predictions length +- `ref_len`: reference length + +The output is in the following format: +```python +{'score': 39.76353643835252, 'counts': [6, 4, 2, 1], 'totals': [10, 8, 6, 4], 'precisions': [60.0, 50.0, 33.333333333333336, 25.0], 'bp': 1.0, 'sys_len': 10, 'ref_len': 7} +``` +The score can take any value between `0.0` and `100.0`, inclusive. + +#### Values from Popular Papers + + +### Examples + +```python +>>> predictions = ["hello there general kenobi", +... "on our way to ankh morpork"] +>>> references = [["hello there general kenobi", "hello there !"], +... ["goodbye ankh morpork", "ankh morpork"]] +>>> sacrebleu = evaluate.load("sacrebleu") +>>> results = sacrebleu.compute(predictions=predictions, +... references=references) +>>> print(list(results.keys())) +['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] +>>> print(round(results["score"], 1)) +39.8 +``` + +## Limitations and Bias +Because what this metric calculates is BLEU scores, it has the same limitations as that metric, except that sacreBLEU is more easily reproducible. + +## Citation +```bibtex +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See the [sacreBLEU README.md file](https://github.com/mjpost/sacreBLEU) for more information. \ No newline at end of file diff --git a/ST/evaluate/metrics/sacrebleu/app.py b/ST/evaluate/metrics/sacrebleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5c77b0098c7f80e74cd3b2197f12b6e087cab35e --- /dev/null +++ b/ST/evaluate/metrics/sacrebleu/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("sacrebleu") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/sacrebleu/requirements.txt b/ST/evaluate/metrics/sacrebleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/ST/evaluate/metrics/sacrebleu/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/ST/evaluate/metrics/sacrebleu/sacrebleu.py b/ST/evaluate/metrics/sacrebleu/sacrebleu.py new file mode 100644 index 0000000000000000000000000000000000000000..6e756f4d4c9bc78390e3bb0d104f0f4515c2a0b7 --- /dev/null +++ b/ST/evaluate/metrics/sacrebleu/sacrebleu.py @@ -0,0 +1,178 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SACREBLEU metric. """ + +import datasets +import sacrebleu as scb +from packaging import version + +import evaluate + + +_CITATION = """\ +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. +Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text. +It also knows all the standard test sets and handles downloading, processing, and tokenization for you. + +See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces BLEU scores along with its sufficient statistics +from a source against one or more references. + +Args: + predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. + references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). + smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: + - `'none'`: no smoothing + - `'floor'`: increment zero counts + - `'add-k'`: increment num/denom by k for n>1 + - `'exp'`: exponential decay + smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). + tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: + - `'none'`: No tokenization. + - `'zh'`: Chinese tokenization. + - `'13a'`: mimics the `mteval-v13a` script from Moses. + - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses + - `'char'`: Language-agnostic character-level tokenization. + - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). + lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. + force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. + use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. + +Returns: + 'score': BLEU score, + 'counts': Counts, + 'totals': Totals, + 'precisions': Precisions, + 'bp': Brevity penalty, + 'sys_len': predictions length, + 'ref_len': reference length, + +Examples: + + Example 1: + >>> predictions = ["hello there general kenobi", "foo bar foobar"] + >>> references = [["hello there general kenobi", "hello there !"], ["foo bar foobar", "foo bar foobar"]] + >>> sacrebleu = evaluate.load("sacrebleu") + >>> results = sacrebleu.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] + >>> print(round(results["score"], 1)) + 100.0 + + Example 2: + >>> predictions = ["hello there general kenobi", + ... "on our way to ankh morpork"] + >>> references = [["hello there general kenobi", "hello there !"], + ... ["goodbye ankh morpork", "ankh morpork"]] + >>> sacrebleu = evaluate.load("sacrebleu") + >>> results = sacrebleu.compute(predictions=predictions, + ... references=references) + >>> print(list(results.keys())) + ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] + >>> print(round(results["score"], 1)) + 39.8 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Sacrebleu(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/mjpost/sacreBLEU", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU"], + reference_urls=[ + "https://github.com/mjpost/sacreBLEU", + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute( + self, + predictions, + references, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + tokenize=None, + use_effective_order=False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + output = scb.corpus_bleu( + predictions, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + **(dict(tokenize=tokenize) if tokenize else {}), + ) + output_dict = { + "score": output.score, + "counts": output.counts, + "totals": output.totals, + "precisions": output.precisions, + "bp": output.bp, + "sys_len": output.sys_len, + "ref_len": output.ref_len, + } + return output_dict diff --git a/ST/evaluate/metrics/sari/README.md b/ST/evaluate/metrics/sari/README.md new file mode 100644 index 0000000000000000000000000000000000000000..eea74d817b8b61a34fe71cf275bcde6e2c547f58 --- /dev/null +++ b/ST/evaluate/metrics/sari/README.md @@ -0,0 +1,146 @@ +--- +title: SARI +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SARI is a metric used for evaluating automatic text simplification systems. + The metric compares the predicted simplified sentences against the reference + and the source sentences. It explicitly measures the goodness of words that are + added, deleted and kept by the system. + Sari = (F1_add + F1_keep + P_del) / 3 + where + F1_add: n-gram F1 score for add operation + F1_keep: n-gram F1 score for keep operation + P_del: n-gram precision score for delete operation + n = 4, as in the original paper. + + This implementation is adapted from Tensorflow's tensor2tensor implementation [3]. + It has two differences with the original GitHub [1] implementation: + (1) Defines 0/0=1 instead of 0 to give higher scores for predictions that match + a target exactly. + (2) Fixes an alleged bug [2] in the keep score computation. + [1] https://github.com/cocoxu/simplification/blob/master/SARI.py + (commit 0210f15) + [2] https://github.com/cocoxu/simplification/issues/6 + [3] https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py +--- + +# Metric Card for SARI + + +## Metric description +SARI (***s**ystem output **a**gainst **r**eferences and against the **i**nput sentence*) is a metric used for evaluating automatic text simplification systems. + +The metric compares the predicted simplified sentences against the reference and the source sentences. It explicitly measures the goodness of words that are added, deleted and kept by the system. + +SARI can be computed as: + +`sari = ( F1_add + F1_keep + P_del) / 3` + +where + +`F1_add` is the n-gram F1 score for add operations + +`F1_keep` is the n-gram F1 score for keep operations + +`P_del` is the n-gram precision score for delete operations + +The number of n grams, `n`, is equal to 4, as in the original paper. + +This implementation is adapted from [Tensorflow's tensor2tensor implementation](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py). +It has two differences with the [original GitHub implementation](https://github.com/cocoxu/simplification/blob/master/SARI.py): + +1) It defines 0/0=1 instead of 0 to give higher scores for predictions that match a target exactly. +2) It fixes an [alleged bug](https://github.com/cocoxu/simplification/issues/6) in the keep score computation. + + + +## How to use + +The metric takes 3 inputs: sources (a list of source sentence strings), predictions (a list of predicted sentence strings) and references (a list of lists of reference sentence strings) + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted."] +predictions=["About 95 you now get in."] +references=[["About 95 species are currently known.","About 95 species are now accepted.","95 species are now accepted."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with the SARI score: + +``` +print(sari_score) +{'sari': 26.953601953601954} +``` + +The range of values for the SARI score is between 0 and 100 -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. + +### Values from popular papers + +The [original paper that proposes the SARI metric](https://aclanthology.org/Q16-1029.pdf) reports scores ranging from 26 to 43 for different simplification systems and different datasets. They also find that the metric ranks all of the simplification systems and human references in the same order as the human assessment used as a comparison, and that it correlates reasonably with human judgments. + +More recent SARI scores for text simplification can be found on leaderboards for datasets such as [TurkCorpus](https://paperswithcode.com/sota/text-simplification-on-turkcorpus) and [Newsela](https://paperswithcode.com/sota/text-simplification-on-newsela). + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted ."] +predictions=["About 95 species are currently accepted ."] +references=[["About 95 species are currently accepted ."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +print(sari_score) +{'sari': 100.0} +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted ."] +predictions=["About 95 you now get in ."] +references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +print(sari_score) +{'sari': 26.953601953601954} +``` + +## Limitations and bias + +SARI is a valuable measure for comparing different text simplification systems as well as one that can assist the iterative development of a system. + +However, while the [original paper presenting SARI](https://aclanthology.org/Q16-1029.pdf) states that it captures "the notion of grammaticality and meaning preservation", this is a difficult claim to empirically validate. + +## Citation + +```bibtex +@inproceedings{xu-etal-2016-optimizing, +title = {Optimizing Statistical Machine Translation for Text Simplification}, +authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, +journal = {Transactions of the Association for Computational Linguistics}, +volume = {4}, +year={2016}, +url = {https://www.aclweb.org/anthology/Q16-1029}, +pages = {401--415}, +} +``` + +## Further References + +- [NLP Progress -- Text Simplification](http://nlpprogress.com/english/simplification.html) +- [Hugging Face Hub -- Text Simplification Models](https://huggingface.co/datasets?filter=task_ids:text-simplification) diff --git a/ST/evaluate/metrics/sari/app.py b/ST/evaluate/metrics/sari/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0666a386de190bab2906c58fef504a081cf52727 --- /dev/null +++ b/ST/evaluate/metrics/sari/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("sari") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/sari/requirements.txt b/ST/evaluate/metrics/sari/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a85cf386b67ac245901a4fe6a22bf7edf6143b1 --- /dev/null +++ b/ST/evaluate/metrics/sari/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu +sacremoses \ No newline at end of file diff --git a/ST/evaluate/metrics/sari/sari.py b/ST/evaluate/metrics/sari/sari.py new file mode 100644 index 0000000000000000000000000000000000000000..7d021184d0f45ae1fb01d86f8c5320a4a62fdf89 --- /dev/null +++ b/ST/evaluate/metrics/sari/sari.py @@ -0,0 +1,289 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SARI metric.""" + +from collections import Counter + +import datasets +import sacrebleu +import sacremoses +from packaging import version + +import evaluate + + +_CITATION = """\ +@inproceedings{xu-etal-2016-optimizing, +title = {Optimizing Statistical Machine Translation for Text Simplification}, +authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, +journal = {Transactions of the Association for Computational Linguistics}, +volume = {4}, +year={2016}, +url = {https://www.aclweb.org/anthology/Q16-1029}, +pages = {401--415}, +} +""" + +_DESCRIPTION = """\ +SARI is a metric used for evaluating automatic text simplification systems. +The metric compares the predicted simplified sentences against the reference +and the source sentences. It explicitly measures the goodness of words that are +added, deleted and kept by the system. +Sari = (F1_add + F1_keep + P_del) / 3 +where +F1_add: n-gram F1 score for add operation +F1_keep: n-gram F1 score for keep operation +P_del: n-gram precision score for delete operation +n = 4, as in the original paper. + +This implementation is adapted from Tensorflow's tensor2tensor implementation [3]. +It has two differences with the original GitHub [1] implementation: + (1) Defines 0/0=1 instead of 0 to give higher scores for predictions that match + a target exactly. + (2) Fixes an alleged bug [2] in the keep score computation. +[1] https://github.com/cocoxu/simplification/blob/master/SARI.py + (commit 0210f15) +[2] https://github.com/cocoxu/simplification/issues/6 +[3] https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py +""" + + +_KWARGS_DESCRIPTION = """ +Calculates sari score (between 0 and 100) given a list of source and predicted +sentences, and a list of lists of reference sentences. +Args: + sources: list of source sentences where each sentence should be a string. + predictions: list of predicted sentences where each sentence should be a string. + references: list of lists of reference sentences where each sentence should be a string. +Returns: + sari: sari score +Examples: + >>> sources=["About 95 species are currently accepted ."] + >>> predictions=["About 95 you now get in ."] + >>> references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] + >>> sari = evaluate.load("sari") + >>> results = sari.compute(sources=sources, predictions=predictions, references=references) + >>> print(results) + {'sari': 26.953601953601954} +""" + + +def SARIngram(sgrams, cgrams, rgramslist, numref): + rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] + rgramcounter = Counter(rgramsall) + + sgramcounter = Counter(sgrams) + sgramcounter_rep = Counter() + for sgram, scount in sgramcounter.items(): + sgramcounter_rep[sgram] = scount * numref + + cgramcounter = Counter(cgrams) + cgramcounter_rep = Counter() + for cgram, ccount in cgramcounter.items(): + cgramcounter_rep[cgram] = ccount * numref + + # KEEP + keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep + keepgramcountergood_rep = keepgramcounter_rep & rgramcounter + keepgramcounterall_rep = sgramcounter_rep & rgramcounter + + keeptmpscore1 = 0 + keeptmpscore2 = 0 + for keepgram in keepgramcountergood_rep: + keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] + # Fix an alleged bug [2] in the keep score computation. + # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] + keeptmpscore2 += keepgramcountergood_rep[keepgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + keepscore_precision = 1 + keepscore_recall = 1 + if len(keepgramcounter_rep) > 0: + keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) + if len(keepgramcounterall_rep) > 0: + # Fix an alleged bug [2] in the keep score computation. + # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) + keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) + keepscore = 0 + if keepscore_precision > 0 or keepscore_recall > 0: + keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) + + # DELETION + delgramcounter_rep = sgramcounter_rep - cgramcounter_rep + delgramcountergood_rep = delgramcounter_rep - rgramcounter + delgramcounterall_rep = sgramcounter_rep - rgramcounter + deltmpscore1 = 0 + deltmpscore2 = 0 + for delgram in delgramcountergood_rep: + deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] + deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + delscore_precision = 1 + if len(delgramcounter_rep) > 0: + delscore_precision = deltmpscore1 / len(delgramcounter_rep) + + # ADDITION + addgramcounter = set(cgramcounter) - set(sgramcounter) + addgramcountergood = set(addgramcounter) & set(rgramcounter) + addgramcounterall = set(rgramcounter) - set(sgramcounter) + + addtmpscore = 0 + for addgram in addgramcountergood: + addtmpscore += 1 + + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + addscore_precision = 1 + addscore_recall = 1 + if len(addgramcounter) > 0: + addscore_precision = addtmpscore / len(addgramcounter) + if len(addgramcounterall) > 0: + addscore_recall = addtmpscore / len(addgramcounterall) + addscore = 0 + if addscore_precision > 0 or addscore_recall > 0: + addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) + + return (keepscore, delscore_precision, addscore) + + +def SARIsent(ssent, csent, rsents): + numref = len(rsents) + + s1grams = ssent.split(" ") + c1grams = csent.split(" ") + s2grams = [] + c2grams = [] + s3grams = [] + c3grams = [] + s4grams = [] + c4grams = [] + + r1gramslist = [] + r2gramslist = [] + r3gramslist = [] + r4gramslist = [] + for rsent in rsents: + r1grams = rsent.split(" ") + r2grams = [] + r3grams = [] + r4grams = [] + r1gramslist.append(r1grams) + for i in range(0, len(r1grams) - 1): + if i < len(r1grams) - 1: + r2gram = r1grams[i] + " " + r1grams[i + 1] + r2grams.append(r2gram) + if i < len(r1grams) - 2: + r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + r3grams.append(r3gram) + if i < len(r1grams) - 3: + r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] + r4grams.append(r4gram) + r2gramslist.append(r2grams) + r3gramslist.append(r3grams) + r4gramslist.append(r4grams) + + for i in range(0, len(s1grams) - 1): + if i < len(s1grams) - 1: + s2gram = s1grams[i] + " " + s1grams[i + 1] + s2grams.append(s2gram) + if i < len(s1grams) - 2: + s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + s3grams.append(s3gram) + if i < len(s1grams) - 3: + s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] + s4grams.append(s4gram) + + for i in range(0, len(c1grams) - 1): + if i < len(c1grams) - 1: + c2gram = c1grams[i] + " " + c1grams[i + 1] + c2grams.append(c2gram) + if i < len(c1grams) - 2: + c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + c3grams.append(c3gram) + if i < len(c1grams) - 3: + c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] + c4grams.append(c4gram) + + (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) + (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) + (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) + (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) + avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 + avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 + avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 + finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 + return finalscore + + +def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): + + # Normalization is requried for the ASSET dataset (one of the primary + # datasets in sentence simplification) to allow using space + # to split the sentence. Even though Wiki-Auto and TURK datasets, + # do not require normalization, we do it for consistency. + # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. + # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 + + if lowercase: + sentence = sentence.lower() + + if tokenizer in ["13a", "intl"]: + if version.parse(sacrebleu.__version__).major >= 2: + normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) + else: + normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) + elif tokenizer == "moses": + normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) + elif tokenizer == "penn": + normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) + else: + normalized_sent = sentence + + if not return_str: + normalized_sent = normalized_sent.split() + + return normalized_sent + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Sari(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "sources": datasets.Value("string", id="sequence"), + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + codebase_urls=[ + "https://github.com/cocoxu/simplification/blob/master/SARI.py", + "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", + ], + reference_urls=["https://www.aclweb.org/anthology/Q16-1029.pdf"], + ) + + def _compute(self, sources, predictions, references): + + if not (len(sources) == len(predictions) == len(references)): + raise ValueError("Sources length must match predictions and references lengths.") + sari_score = 0 + for src, pred, refs in zip(sources, predictions, references): + sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) + sari_score = sari_score / len(predictions) + return {"sari": 100 * sari_score} diff --git a/ST/evaluate/metrics/seqeval/README.md b/ST/evaluate/metrics/seqeval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e4442315f079863e802dfc7413e0166ac7d01071 --- /dev/null +++ b/ST/evaluate/metrics/seqeval/README.md @@ -0,0 +1,164 @@ +--- +title: seqeval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + seqeval is a Python framework for sequence labeling evaluation. + seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + + This is well-tested by using the Perl script conlleval, which can be used for + measuring the performance of a system that has processed the CoNLL-2000 shared task data. + + seqeval supports following formats: + IOB1 + IOB2 + IOE1 + IOE2 + IOBES + + See the [README.md] file at https://github.com/chakki-works/seqeval for more information. +--- + +# Metric Card for seqeval + +## Metric description + +seqeval is a Python framework for sequence labeling evaluation. seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + + +## How to use + +Seqeval produces labelling scores along with its sufficient statistics from a source against one or more references. + +It takes two mandatory arguments: + +`predictions`: a list of lists of predicted labels, i.e. estimated targets as returned by a tagger. + +`references`: a list of lists of reference labels, i.e. the ground truth/target values. + +It can also take several optional arguments: + +`suffix` (boolean): `True` if the IOB tag is a suffix (after type) instead of a prefix (before type), `False` otherwise. The default value is `False`, i.e. the IOB tag is a prefix (before type). + +`scheme`: the target tagging scheme, which can be one of [`IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `BILOU`]. The default value is `None`. + +`mode`: whether to count correct entity labels with incorrect I/B tags as true positives or not. If you want to only count exact matches, pass `mode="strict"` and a specific `scheme` value. The default is `None`. + +`sample_weight`: An array-like of shape (n_samples,) that provides weights for individual samples. The default is `None`. + +`zero_division`: Which value to substitute as a metric value when encountering zero division. Should be one of [`0`,`1`,`"warn"`]. `"warn"` acts as `0`, but the warning is raised. + + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +``` + +## Output values + +This metric returns a dictionary with a summary of scores for overall and per type: + +Overall: + +`accuracy`: the average [accuracy](https://huggingface.co/metrics/accuracy), on a scale between 0.0 and 1.0. + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), which is the harmonic mean of the precision and recall. It also has a scale of 0.0 to 1.0. + +Per type (e.g. `MISC`, `PER`, `LOC`,...): + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), on a scale between 0.0 and 1.0. + + +### Values from popular papers +The 1995 "Text Chunking using Transformation-Based Learning" [paper](https://aclanthology.org/W95-0107) reported a baseline recall of 81.9% and a precision of 78.2% using non Deep Learning-based methods. + +More recently, seqeval continues being used for reporting performance on tasks such as [named entity detection](https://www.mdpi.com/2306-5729/6/8/84/htm) and [information extraction](https://ieeexplore.ieee.org/abstract/document/9697942/). + + +## Examples + +Maximal values (full match) : + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 1.0, 'overall_recall': 1.0, 'overall_f1': 1.0, 'overall_accuracy': 1.0} + +``` + +Minimal values (no match): + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'B-MISC', 'I-MISC'], ['B-PER', 'I-PER', 'O']] +>>> references = [['B-MISC', 'O', 'O'], ['I-PER', '0', 'I-PER']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 2}, '_': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'overall_precision': 0.0, 'overall_recall': 0.0, 'overall_f1': 0.0, 'overall_accuracy': 0.0} +``` + +Partial match: + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 0.5, 'overall_recall': 0.5, 'overall_f1': 0.5, 'overall_accuracy': 0.8} +``` + +## Limitations and bias + +seqeval supports following IOB formats (short for inside, outside, beginning) : `IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `IOBES` (only in strict mode) and `BILOU` (only in strict mode). + +For more information about IOB formats, refer to the [Wikipedia page](https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging)) and the description of the [CoNLL-2000 shared task](https://aclanthology.org/W02-2024). + + +## Citation + +```bibtex +@inproceedings{ramshaw-marcus-1995-text, + title = "Text Chunking using Transformation-Based Learning", + author = "Ramshaw, Lance and + Marcus, Mitch", + booktitle = "Third Workshop on Very Large Corpora", + year = "1995", + url = "https://www.aclweb.org/anthology/W95-0107", +} +``` + +```bibtex +@misc{seqeval, + title={{seqeval}: A Python framework for sequence labeling evaluation}, + url={https://github.com/chakki-works/seqeval}, + note={Software available from https://github.com/chakki-works/seqeval}, + author={Hiroki Nakayama}, + year={2018}, +} +``` + +## Further References +- [README for seqeval at GitHub](https://github.com/chakki-works/seqeval) +- [CoNLL-2000 shared task](https://www.clips.uantwerpen.be/conll2002/ner/bin/conlleval.txt) diff --git a/ST/evaluate/metrics/seqeval/app.py b/ST/evaluate/metrics/seqeval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..7c05d59ec96e2789c3850ce0d656b932564d34af --- /dev/null +++ b/ST/evaluate/metrics/seqeval/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("seqeval") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/seqeval/requirements.txt b/ST/evaluate/metrics/seqeval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac681b46ff66dba1b0520056ef07c2fd32566d14 --- /dev/null +++ b/ST/evaluate/metrics/seqeval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +seqeval \ No newline at end of file diff --git a/ST/evaluate/metrics/seqeval/seqeval.py b/ST/evaluate/metrics/seqeval/seqeval.py new file mode 100644 index 0000000000000000000000000000000000000000..252d16b6c6f4f45ae41bb982c00771ae3d707441 --- /dev/null +++ b/ST/evaluate/metrics/seqeval/seqeval.py @@ -0,0 +1,164 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" seqeval metric. """ + +import importlib +from typing import List, Optional, Union + +import datasets +from seqeval.metrics import accuracy_score, classification_report + +import evaluate + + +_CITATION = """\ +@inproceedings{ramshaw-marcus-1995-text, + title = "Text Chunking using Transformation-Based Learning", + author = "Ramshaw, Lance and + Marcus, Mitch", + booktitle = "Third Workshop on Very Large Corpora", + year = "1995", + url = "https://www.aclweb.org/anthology/W95-0107", +} +@misc{seqeval, + title={{seqeval}: A Python framework for sequence labeling evaluation}, + url={https://github.com/chakki-works/seqeval}, + note={Software available from https://github.com/chakki-works/seqeval}, + author={Hiroki Nakayama}, + year={2018}, +} +""" + +_DESCRIPTION = """\ +seqeval is a Python framework for sequence labeling evaluation. +seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + +This is well-tested by using the Perl script conlleval, which can be used for +measuring the performance of a system that has processed the CoNLL-2000 shared task data. + +seqeval supports following formats: +IOB1 +IOB2 +IOE1 +IOE2 +IOBES + +See the [README.md] file at https://github.com/chakki-works/seqeval for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces labelling scores along with its sufficient statistics +from a source against one or more references. + +Args: + predictions: List of List of predicted labels (Estimated targets as returned by a tagger) + references: List of List of reference labels (Ground truth (correct) target values) + suffix: True if the IOB prefix is after type, False otherwise. default: False + scheme: Specify target tagging scheme. Should be one of ["IOB1", "IOB2", "IOE1", "IOE2", "IOBES", "BILOU"]. + default: None + mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not. + If you want to only count exact matches, pass mode="strict". default: None. + sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None + zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + 'scores': dict. Summary of the scores for overall and per type + Overall: + 'accuracy': accuracy, + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure, + Per type: + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure +Examples: + + >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] + >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] + >>> seqeval = evaluate.load("seqeval") + >>> results = seqeval.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy'] + >>> print(results["overall_f1"]) + 0.5 + >>> print(results["PER"]["f1"]) + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Seqeval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/chakki-works/seqeval", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + } + ), + codebase_urls=["https://github.com/chakki-works/seqeval"], + reference_urls=["https://github.com/chakki-works/seqeval"], + ) + + def _compute( + self, + predictions, + references, + suffix: bool = False, + scheme: Optional[str] = None, + mode: Optional[str] = None, + sample_weight: Optional[List[int]] = None, + zero_division: Union[str, int] = "warn", + ): + if scheme is not None: + try: + scheme_module = importlib.import_module("seqeval.scheme") + scheme = getattr(scheme_module, scheme) + except AttributeError: + raise ValueError(f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}") + report = classification_report( + y_true=references, + y_pred=predictions, + suffix=suffix, + output_dict=True, + scheme=scheme, + mode=mode, + sample_weight=sample_weight, + zero_division=zero_division, + ) + report.pop("macro avg") + report.pop("weighted avg") + overall_score = report.pop("micro avg") + + scores = { + type_name: { + "precision": score["precision"], + "recall": score["recall"], + "f1": score["f1-score"], + "number": score["support"], + } + for type_name, score in report.items() + } + scores["overall_precision"] = overall_score["precision"] + scores["overall_recall"] = overall_score["recall"] + scores["overall_f1"] = overall_score["f1-score"] + scores["overall_accuracy"] = accuracy_score(y_true=references, y_pred=predictions) + + return scores diff --git a/ST/evaluate/metrics/smape/README.md b/ST/evaluate/metrics/smape/README.md new file mode 100644 index 0000000000000000000000000000000000000000..904e17ff28b5c38a27953d130f359e2d37ede60c --- /dev/null +++ b/ST/evaluate/metrics/smape/README.md @@ -0,0 +1,112 @@ +--- +title: sMAPE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error difference between the predicted and actual values defined by Chen and Yang (2004). +--- + +# Metric Card for sMAPE + + +## Metric Description + +Symmetric Mean Absolute Error (sMAPE) is the symmetric mean of the percentage error of difference between the predicted $x_i$ and actual $y_i$ numeric values: + +![image](https://user-images.githubusercontent.com/8100/200009801-ae8be6c8-facf-401b-8df0-3f80a458b9f4.png) + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> smape_metric = evaluate.load("smape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = smape_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each sMAPE `float` value ranges from `0.0` to `2.0`, with the best value being 0.0. + +Output Example(s): +```python +{'smape': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'smape': array([0.5, 1.5 ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> smape_metric = evaluate.load("smape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = smape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'smape': 0.5787...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> smape_metric = evaluate.load("smape", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> results = smape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'smape': 0.8874...} +>>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'smape': array([1.3749..., 0.4])} +``` + +## Limitations and Bias +This metric is called a measure of "percentage error" even though there is no multiplier of 100. The range is between (0, 2) with it being two when the target and prediction are both zero. + +## Citation(s) + +```bibtex +@article{article, + author = {Chen, Zhuo and Yang, Yuhong}, + year = {2004}, + month = {04}, + pages = {}, + title = {Assessing forecast accuracy measures} +} +``` + +## Further References +- [Symmetric Mean absolute percentage error - Wikipedia](https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error) diff --git a/ST/evaluate/metrics/smape/app.py b/ST/evaluate/metrics/smape/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e08d5961ebcda924f2cac983b4f01ef9ad1b37 --- /dev/null +++ b/ST/evaluate/metrics/smape/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("smape") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/smape/requirements.txt b/ST/evaluate/metrics/smape/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/ST/evaluate/metrics/smape/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/ST/evaluate/metrics/smape/smape.py b/ST/evaluate/metrics/smape/smape.py new file mode 100644 index 0000000000000000000000000000000000000000..b8292c37c20f427f9621445e44d59b3fdf79053c --- /dev/null +++ b/ST/evaluate/metrics/smape/smape.py @@ -0,0 +1,158 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""sMAPE - Symmetric Mean Absolute Percentage Error Metric""" + +import datasets +import numpy as np +from sklearn.metrics._regression import _check_reg_targets +from sklearn.utils.validation import check_consistent_length + +import evaluate + + +_CITATION = """\ +@article{article, + author = {Chen, Zhuo and Yang, Yuhong}, + year = {2004}, + month = {04}, + pages = {}, + title = {Assessing forecast accuracy measures} +} +""" + +_DESCRIPTION = """\ +Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error +difference between the predicted and actual values as defined by Chen and Yang (2004), +based on the metric by Armstrong (1985) and Makridakis (1993). +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + smape : symmetric mean absolute percentage error. + If multioutput is "raw_values", then symmetric mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + sMAPE output is non-negative floating point in the range (0, 2). The best value is 0.0. +Examples: + + >>> smape_metric = evaluate.load("smape") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = smape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'smape': 0.5787878787878785} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> smape_metric = evaluate.load("smape", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0.1, 2], [-1, 2], [8, -5]] + >>> results = smape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'smape': 0.49696969558995985} + >>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'smape': array([0.48888889, 0.50505051])} +""" + + +def symmetric_mean_absolute_percentage_error(y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"): + """Symmetric Mean absolute percentage error (sMAPE) metric using sklearn's api and helpers. + + Parameters + ---------- + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + multioutput : {'raw_values', 'uniform_average'} or array-like + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + If input is list then the shape must be (n_outputs,). + 'raw_values' : + Returns a full set of errors in case of multioutput input. + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + Returns + ------- + loss : float or ndarray of floats + If multioutput is 'raw_values', then mean absolute percentage error + is returned for each output separately. + If multioutput is 'uniform_average' or an ndarray of weights, then the + weighted average of all output errors is returned. + sMAPE output is non-negative floating point. The best value is 0.0. + """ + y_type, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput) + check_consistent_length(y_true, y_pred, sample_weight) + epsilon = np.finfo(np.float64).eps + smape = 2 * np.abs(y_pred - y_true) / (np.maximum(np.abs(y_true), epsilon) + np.maximum(np.abs(y_pred), epsilon)) + output_errors = np.average(smape, weights=sample_weight, axis=0) + if isinstance(multioutput, str): + if multioutput == "raw_values": + return output_errors + elif multioutput == "uniform_average": + # pass None as weights to np.average: uniform mean + multioutput = None + + return np.average(output_errors, weights=multioutput) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Smape(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=["https://robjhyndman.com/hyndsight/smape/"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + smape_score = symmetric_mean_absolute_percentage_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + return {"smape": smape_score} diff --git a/ST/evaluate/metrics/spearmanr/README.md b/ST/evaluate/metrics/spearmanr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bc42bdfa7b485f61a39fd790c5f824b9c029e2a7 --- /dev/null +++ b/ST/evaluate/metrics/spearmanr/README.md @@ -0,0 +1,146 @@ +--- +title: Spearman Correlation Coefficient Metric +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The Spearman rank-order correlation coefficient is a measure of the + relationship between two datasets. Like other correlation coefficients, + this one varies between -1 and +1 with 0 implying no correlation. + Positive correlations imply that as data in dataset x increases, so + does data in dataset y. Negative correlations imply that as x increases, + y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + + Unlike the Pearson correlation, the Spearman correlation does not + assume that both datasets are normally distributed. + + The p-value roughly indicates the probability of an uncorrelated system + producing datasets that have a Spearman correlation at least as extreme + as the one computed from these datasets. The p-values are not entirely + reliable but are probably reasonable for datasets larger than 500 or so. +--- + +# Metric Card for Spearman Correlation Coefficient Metric (spearmanr) + + +## Metric Description +The Spearman rank-order correlation coefficient is a measure of the +relationship between two datasets. Like other correlation coefficients, +this one varies between -1 and +1 with 0 implying no correlation. +Positive correlations imply that as data in dataset x increases, so +does data in dataset y. Negative correlations imply that as x increases, +y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + +Unlike the Pearson correlation, the Spearman correlation does not +assume that both datasets are normally distributed. + +The p-value roughly indicates the probability of an uncorrelated system +producing datasets that have a Spearman correlation at least as extreme +as the one computed from these datasets. The p-values are not entirely +reliable but are probably reasonable for datasets larger than 500 or so. + + +## How to Use +At minimum, this metric only requires a `list` of predictions and a `list` of references: + +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) +>>> print(results) +{'spearmanr': -0.7} +``` + +### Inputs +- **`predictions`** (`list` of `float`): Predicted labels, as returned by a model. +- **`references`** (`list` of `float`): Ground truth labels. +- **`return_pvalue`** (`bool`): If `True`, returns the p-value. If `False`, returns + only the spearmanr score. Defaults to `False`. + +### Output Values +- **`spearmanr`** (`float`): Spearman correlation coefficient. +- **`p-value`** (`float`): p-value. **Note**: is only returned + if `return_pvalue=True` is input. + +If `return_pvalue=False`, the output is a `dict` with one value, as below: +```python +{'spearmanr': -0.7} +``` + +Otherwise, if `return_pvalue=True`, the output is a `dict` containing a the `spearmanr` value as well as the corresponding `pvalue`: +```python +{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873} +``` + +Spearman rank-order correlations can take on any value from `-1` to `1`, inclusive. + +The p-values can take on any value from `0` to `1`, inclusive. + +#### Values from Popular Papers + + +### Examples +A basic example: +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) +>>> print(results) +{'spearmanr': -0.7} +``` + +The same example, but that also returns the pvalue: +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4], return_pvalue=True) +>>> print(results) +{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873 +>>> print(results['spearmanr']) +-0.7 +>>> print(results['spearmanr_pvalue']) +0.1881204043741873 +``` + +## Limitations and Bias + + +## Citation +```bibtex +@book{kokoska2000crc, + title={CRC standard probability and statistics tables and formulae}, + author={Kokoska, Stephen and Zwillinger, Daniel}, + year={2000}, + publisher={Crc Press} +} +@article{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +``` + +## Further References +*Add any useful further references.* diff --git a/ST/evaluate/metrics/spearmanr/app.py b/ST/evaluate/metrics/spearmanr/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6cccd91ec9e4b6e27579abad648291d0d6b339ff --- /dev/null +++ b/ST/evaluate/metrics/spearmanr/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("spearmanr") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/spearmanr/requirements.txt b/ST/evaluate/metrics/spearmanr/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/ST/evaluate/metrics/spearmanr/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/ST/evaluate/metrics/spearmanr/spearmanr.py b/ST/evaluate/metrics/spearmanr/spearmanr.py new file mode 100644 index 0000000000000000000000000000000000000000..44d372638f51aab0b358217cd059dd1353a7f494 --- /dev/null +++ b/ST/evaluate/metrics/spearmanr/spearmanr.py @@ -0,0 +1,120 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Spearman correlation coefficient metric.""" + +import datasets +from scipy.stats import spearmanr + +import evaluate + + +_DESCRIPTION = """ +The Spearman rank-order correlation coefficient is a measure of the +relationship between two datasets. Like other correlation coefficients, +this one varies between -1 and +1 with 0 implying no correlation. +Positive correlations imply that as data in dataset x increases, so +does data in dataset y. Negative correlations imply that as x increases, +y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + +Unlike the Pearson correlation, the Spearman correlation does not +assume that both datasets are normally distributed. + +The p-value roughly indicates the probability of an uncorrelated system +producing datasets that have a Spearman correlation at least as extreme +as the one computed from these datasets. The p-values are not entirely +reliable but are probably reasonable for datasets larger than 500 or so. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`List[float]`): Predicted labels, as returned by a model. + references (`List[float]`): Ground truth labels. + return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns + only the spearmanr score. Defaults to `False`. +Returns: + spearmanr (`float`): Spearman correlation coefficient. + p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input. +Examples: + Example 1: + >>> spearmanr_metric = evaluate.load("spearmanr") + >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) + >>> print(results) + {'spearmanr': -0.7} + + Example 2: + >>> spearmanr_metric = evaluate.load("spearmanr") + >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], + ... predictions=[10, 9, 2.5, 6, 4], + ... return_pvalue=True) + >>> print(results['spearmanr']) + -0.7 + >>> print(round(results['spearmanr_pvalue'], 2)) + 0.19 +""" + +_CITATION = r"""\ +@book{kokoska2000crc, + title={CRC standard probability and statistics tables and formulae}, + author={Kokoska, Stephen and Zwillinger, Daniel}, + year={2000}, + publisher={Crc Press} +} +@article{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Spearmanr(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + ), + reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"], + ) + + def _compute(self, predictions, references, return_pvalue=False): + results = spearmanr(references, predictions) + if return_pvalue: + return {"spearmanr": results[0], "spearmanr_pvalue": results[1]} + else: + return {"spearmanr": results[0]} diff --git a/ST/evaluate/metrics/squad/README.md b/ST/evaluate/metrics/squad/README.md new file mode 100644 index 0000000000000000000000000000000000000000..08e030c599698bd0bdf7aa986f1bc0c14bb792cf --- /dev/null +++ b/ST/evaluate/metrics/squad/README.md @@ -0,0 +1,110 @@ +--- +title: SQuAD +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD). + + Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by + crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, + from the corresponding reading passage, or the question might be unanswerable. +--- + +# Metric Card for SQuAD + +## Metric description +This metric wraps the official scoring script for version 1 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad). + +SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. + +## How to use + +The metric takes two files or two lists of question-answers dictionaries as inputs : one with the predictions of the model and the other with the references to be compared to: + +```python +from evaluate import load +squad_metric = load("squad") +results = squad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with two values: the average exact match score and the average [F1 score](https://huggingface.co/metrics/f1). + +``` +{'exact_match': 100.0, 'f1': 100.0} +``` + +The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. + +The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +### Values from popular papers +The [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an F1 score of 51.0% and an Exact Match score of 40.0%. They also report that human performance on the dataset represents an F1 score of 90.5% and an Exact Match score of 80.3%. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). + +## Examples + +Maximal values for both exact match and F1 (perfect match): + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 100.0, 'f1': 100.0} +``` + +Minimal values for both exact match and F1 (no match): + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 0.0, 'f1': 0.0} +``` + +Partial match (2 out of 3 answers correct) : + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b'}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['Beyoncé and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 66.66666666666667, 'f1': 66.66666666666667} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as [SQuAD v.1 dataset](https://huggingface.co/datasets/squad). + +The SQuAD dataset does contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. + + +## Citation + + @inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} + } + +## Further References + +- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) +- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/ST/evaluate/metrics/squad/app.py b/ST/evaluate/metrics/squad/app.py new file mode 100644 index 0000000000000000000000000000000000000000..7e22ac6a4baf025348645eb91d8f48fd206f715a --- /dev/null +++ b/ST/evaluate/metrics/squad/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("squad") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/squad/compute_score.py b/ST/evaluate/metrics/squad/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..e60acfd1044319e43a59ffe8db75e63b68785ba7 --- /dev/null +++ b/ST/evaluate/metrics/squad/compute_score.py @@ -0,0 +1,92 @@ +""" Official evaluation script for v1.1 of the SQuAD dataset. """ + +import argparse +import json +import re +import string +import sys +from collections import Counter + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1_score(prediction, ground_truth): + prediction_tokens = normalize_answer(prediction).split() + ground_truth_tokens = normalize_answer(ground_truth).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def compute_score(dataset, predictions): + f1 = exact_match = total = 0 + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + print(message, file=sys.stderr) + continue + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) + + exact_match = 100.0 * exact_match / total + f1 = 100.0 * f1 / total + + return {"exact_match": exact_match, "f1": f1} + + +if __name__ == "__main__": + expected_version = "1.1" + parser = argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version) + parser.add_argument("dataset_file", help="Dataset file") + parser.add_argument("prediction_file", help="Prediction File") + args = parser.parse_args() + with open(args.dataset_file) as dataset_file: + dataset_json = json.load(dataset_file) + if dataset_json["version"] != expected_version: + print( + "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"], + file=sys.stderr, + ) + dataset = dataset_json["data"] + with open(args.prediction_file) as prediction_file: + predictions = json.load(prediction_file) + print(json.dumps(compute_score(dataset, predictions))) diff --git a/ST/evaluate/metrics/squad/requirements.txt b/ST/evaluate/metrics/squad/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/squad/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/squad/squad.py b/ST/evaluate/metrics/squad/squad.py new file mode 100644 index 0000000000000000000000000000000000000000..84658b125f47aed592b6da4659ec60b22e02fe34 --- /dev/null +++ b/ST/evaluate/metrics/squad/squad.py @@ -0,0 +1,111 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SQuAD metric. """ + +import datasets + +import evaluate + +from .compute_score import compute_score + + +_CITATION = """\ +@inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD). + +Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by +crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, +from the corresponding reading passage, or the question might be unanswerable. +""" + +_KWARGS_DESCRIPTION = """ +Computes SQuAD scores (F1 and EM). +Args: + predictions: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair as given in the references (see below) + - 'prediction_text': the text of the answer + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a Dict in the SQuAD dataset format + { + 'text': list of possible texts for the answer, as a list of strings + 'answer_start': list of start positions for the answer, as a list of ints + } + Note that answer_start values are not taken into account to compute the metric. +Returns: + 'exact_match': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer +Examples: + + >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] + >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] + >>> squad_metric = evaluate.load("squad") + >>> results = squad_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 100.0, 'f1': 100.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Squad(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")}, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + { + "text": datasets.Value("string"), + "answer_start": datasets.Value("int32"), + } + ), + }, + } + ), + codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + ) + + def _compute(self, predictions, references): + pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} + dataset = [ + { + "paragraphs": [ + { + "qas": [ + { + "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], + "id": ref["id"], + } + for ref in references + ] + } + ] + } + ] + score = compute_score(dataset=dataset, predictions=pred_dict) + return score diff --git a/ST/evaluate/metrics/squad_v2/README.md b/ST/evaluate/metrics/squad_v2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6c4a7e2435106940f05565b5523f3dc53c98ed1d --- /dev/null +++ b/ST/evaluate/metrics/squad_v2/README.md @@ -0,0 +1,142 @@ +--- +title: SQuAD v2 +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 2 of the Stanford Question Answering Dataset (SQuAD). + + Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by + crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, + from the corresponding reading passage, or the question might be unanswerable. + + SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions + written adversarially by crowdworkers to look similar to answerable ones. + To do well on SQuAD2.0, systems must not only answer questions when possible, but also + determine when no answer is supported by the paragraph and abstain from answering. +--- + +# Metric Card for SQuAD v2 + +## Metric description +This metric wraps the official scoring script for version 2 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad_v2). + +SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. + +SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. + +## How to use + +The metric takes two files or two lists - one representing model predictions and the other the references to compare them to. + +*Predictions* : List of triple for question-answers to score with the following key-value pairs: +* `'id'`: the question-answer identification field of the question and answer pair +* `'prediction_text'` : the text of the answer +* `'no_answer_probability'` : the probability that the question has no answer + +*References*: List of question-answers dictionaries with the following key-value pairs: +* `'id'`: id of the question-answer pair (see above), +* `'answers'`: a list of Dict {'text': text of the answer as a string} +* `'no_answer_threshold'`: the probability threshold to decide that a question has no answer. + +```python +from evaluate import load +squad_metric = load("squad_v2") +results = squad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with 13 values: +* `'exact'`: Exact match (the normalized answer exactly match the gold answer) (see the `exact_match` metric (forthcoming)) +* `'f1'`: The average F1-score of predicted tokens versus the gold answer (see the [F1 score](https://huggingface.co/metrics/f1) metric) +* `'total'`: Number of scores considered +* `'HasAns_exact'`: Exact match (the normalized answer exactly match the gold answer) +* `'HasAns_f1'`: The F-score of predicted tokens versus the gold answer +* `'HasAns_total'`: How many of the questions have answers +* `'NoAns_exact'`: Exact match (the normalized answer exactly match the gold answer) +* `'NoAns_f1'`: The F-score of predicted tokens versus the gold answer +* `'NoAns_total'`: How many of the questions have no answers +* `'best_exact'` : Best exact match (with varying threshold) +* `'best_exact_thresh'`: No-answer probability threshold associated to the best exact match +* `'best_f1'`: Best F1 score (with varying threshold) +* `'best_f1_thresh'`: No-answer probability threshold associated to the best F1 + + +The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. + +The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +The range of `total` depends on the length of predictions/references: its minimal value is 0, and maximal value is the total number of questions in the predictions and references. + +### Values from popular papers +The [SQuAD v2 paper](https://arxiv.org/pdf/1806.03822.pdf) reported an F1 score of 66.3% and an Exact Match score of 63.4%. +They also report that human performance on the dataset represents an F1 score of 89.5% and an Exact Match score of 86.9%. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). + +## Examples + +Maximal values for both exact match and F1 (perfect match): + +```python +from evaluate import load +squad_v2_ metric = load("squad_v2") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} +``` + +Minimal values for both exact match and F1 (no match): + +```python +from evaluate import load +squad_metric = load("squad_v2") +predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 0.0, 'f1': 0.0, 'total': 1, 'HasAns_exact': 0.0, 'HasAns_f1': 0.0, 'HasAns_total': 1, 'best_exact': 0.0, 'best_exact_thresh': 0.0, 'best_f1': 0.0, 'best_f1_thresh': 0.0} +``` + +Partial match (2 out of 3 answers correct) : + +```python +from evaluate import load +squad_metric = load("squad_v2") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b', 'no_answer_probability': 0.}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['Beyoncé and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 66.66666666666667, 'f1': 66.66666666666667, 'total': 3, 'HasAns_exact': 66.66666666666667, 'HasAns_f1': 66.66666666666667, 'HasAns_total': 3, 'best_exact': 66.66666666666667, 'best_exact_thresh': 0.0, 'best_f1': 66.66666666666667, 'best_f1_thresh': 0.0} +``` + +## Limitations and bias +This metric works only with the datasets in the same format as the [SQuAD v.2 dataset](https://huggingface.co/datasets/squad_v2). + +The SQuAD datasets do contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. + + +## Citation + +```bibtex +@inproceedings{Rajpurkar2018SQuAD2, +title={Know What You Don't Know: Unanswerable Questions for SQuAD}, +author={Pranav Rajpurkar and Jian Zhang and Percy Liang}, +booktitle={ACL 2018}, +year={2018} +} +``` + +## Further References + +- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) +- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/ST/evaluate/metrics/squad_v2/app.py b/ST/evaluate/metrics/squad_v2/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6e6ccafc20ab508661ed8a2964e51772e7012fc4 --- /dev/null +++ b/ST/evaluate/metrics/squad_v2/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("squad_v2") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/squad_v2/compute_score.py b/ST/evaluate/metrics/squad_v2/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..3b512ae92484772a8d10bdf43f4df68520cdf3ed --- /dev/null +++ b/ST/evaluate/metrics/squad_v2/compute_score.py @@ -0,0 +1,323 @@ +"""Official evaluation script for SQuAD version 2.0. + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" +import argparse +import collections +import json +import os +import re +import string +import sys + +import numpy as np + + +ARTICLES_REGEX = re.compile(r"\b(a|an|the)\b", re.UNICODE) + +OPTS = None + + +def parse_args(): + parser = argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.") + parser.add_argument("data_file", metavar="data.json", help="Input data JSON file.") + parser.add_argument("pred_file", metavar="pred.json", help="Model predictions.") + parser.add_argument( + "--out-file", "-o", metavar="eval.json", help="Write accuracy metrics to file (default is stdout)." + ) + parser.add_argument( + "--na-prob-file", "-n", metavar="na_prob.json", help="Model estimates of probability of no answer." + ) + parser.add_argument( + "--na-prob-thresh", + "-t", + type=float, + default=1.0, + help='Predict "" if no-answer probability exceeds this (default = 1.0).', + ) + parser.add_argument( + "--out-image-dir", "-p", metavar="out_images", default=None, help="Save precision-recall curves to directory." + ) + parser.add_argument("--verbose", "-v", action="store_true") + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + return parser.parse_args() + + +def make_qid_to_has_ans(dataset): + qid_to_has_ans = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid_to_has_ans[qa["id"]] = bool(qa["answers"]["text"]) + return qid_to_has_ans + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return ARTICLES_REGEX.sub(" ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_f1(a_gold, a_pred): + gold_toks = get_tokens(a_gold) + pred_toks = get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def get_raw_scores(dataset, preds): + exact_scores = {} + f1_scores = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid = qa["id"] + gold_answers = [t for t in qa["answers"]["text"] if normalize_answer(t)] + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [""] + if qid not in preds: + print(f"Missing prediction for {qid}") + continue + a_pred = preds[qid] + # Take max over all gold answers + exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) + f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) + return exact_scores, f1_scores + + +def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def make_eval_dict(exact_scores, f1_scores, qid_list=None): + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores.values()) / total), + ("f1", 100.0 * sum(f1_scores.values()) / total), + ("total", total), + ] + ) + else: + total = len(qid_list) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ("total", total), + ] + ) + + +def merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval[f"{prefix}_{k}"] = new_eval[k] + + +def plot_pr_curve(precisions, recalls, out_image, title): + plt.step(recalls, precisions, color="b", alpha=0.2, where="post") + plt.fill_between(recalls, precisions, step="post", alpha=0.2, color="b") + plt.xlabel("Recall") + plt.ylabel("Precision") + plt.xlim([0.0, 1.05]) + plt.ylim([0.0, 1.05]) + plt.title(title) + plt.savefig(out_image) + plt.clf() + + +def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, out_image=None, title=None): + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + true_pos = 0.0 + cur_p = 1.0 + cur_r = 0.0 + precisions = [1.0] + recalls = [0.0] + avg_prec = 0.0 + for i, qid in enumerate(qid_list): + if qid_to_has_ans[qid]: + true_pos += scores[qid] + cur_p = true_pos / float(i + 1) + cur_r = true_pos / float(num_true_pos) + if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]: + # i.e., if we can put a threshold after this point + avg_prec += cur_p * (cur_r - recalls[-1]) + precisions.append(cur_p) + recalls.append(cur_r) + if out_image: + plot_pr_curve(precisions, recalls, out_image, title) + return {"ap": 100.0 * avg_prec} + + +def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, out_image_dir): + if out_image_dir and not os.path.exists(out_image_dir): + os.makedirs(out_image_dir) + num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) + if num_true_pos == 0: + return + pr_exact = make_precision_recall_eval( + exact_raw, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_exact.png"), + title="Precision-Recall curve for Exact Match score", + ) + pr_f1 = make_precision_recall_eval( + f1_raw, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_f1.png"), + title="Precision-Recall curve for F1 score", + ) + oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} + pr_oracle = make_precision_recall_eval( + oracle_scores, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_oracle.png"), + title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)", + ) + merge_eval(main_eval, pr_exact, "pr_exact") + merge_eval(main_eval, pr_f1, "pr_f1") + merge_eval(main_eval, pr_oracle, "pr_oracle") + + +def histogram_na_prob(na_probs, qid_list, image_dir, name): + if not qid_list: + return + x = [na_probs[k] for k in qid_list] + weights = np.ones_like(x) / float(len(x)) + plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0)) + plt.xlabel("Model probability of no-answer") + plt.ylabel("Proportion of dataset") + plt.title(f"Histogram of no-answer probability: {name}") + plt.savefig(os.path.join(image_dir, f"na_prob_hist_{name}.png")) + plt.clf() + + +def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for i, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) + main_eval["best_exact"] = best_exact + main_eval["best_exact_thresh"] = exact_thresh + main_eval["best_f1"] = best_f1 + main_eval["best_f1_thresh"] = f1_thresh + + +def main(): + with open(OPTS.data_file) as f: + dataset_json = json.load(f) + dataset = dataset_json["data"] + with open(OPTS.pred_file) as f: + preds = json.load(f) + if OPTS.na_prob_file: + with open(OPTS.na_prob_file) as f: + na_probs = json.load(f) + else: + na_probs = {k: 0.0 for k in preds} + qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + exact_raw, f1_raw = get_raw_scores(dataset, preds) + exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) + f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) + out_eval = make_eval_dict(exact_thresh, f1_thresh) + if has_ans_qids: + has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) + merge_eval(out_eval, has_ans_eval, "HasAns") + if no_ans_qids: + no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + merge_eval(out_eval, no_ans_eval, "NoAns") + if OPTS.na_prob_file: + find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans) + if OPTS.na_prob_file and OPTS.out_image_dir: + run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, OPTS.out_image_dir) + histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, "hasAns") + histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, "noAns") + if OPTS.out_file: + with open(OPTS.out_file, "w") as f: + json.dump(out_eval, f) + else: + print(json.dumps(out_eval, indent=2)) + + +if __name__ == "__main__": + OPTS = parse_args() + if OPTS.out_image_dir: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + main() diff --git a/ST/evaluate/metrics/squad_v2/requirements.txt b/ST/evaluate/metrics/squad_v2/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/squad_v2/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/squad_v2/squad_v2.py b/ST/evaluate/metrics/squad_v2/squad_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..cb9ba1ae85756f2b1d2bd1713c87c92b87e4c0f8 --- /dev/null +++ b/ST/evaluate/metrics/squad_v2/squad_v2.py @@ -0,0 +1,137 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SQuAD v2 metric. """ + +import datasets + +import evaluate + +from .compute_score import ( + apply_no_ans_threshold, + find_all_best_thresh, + get_raw_scores, + make_eval_dict, + make_qid_to_has_ans, + merge_eval, +) + + +_CITATION = """\ +@inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 2 of the Stanford Question +Answering Dataset (SQuAD). + +Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by +crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, +from the corresponding reading passage, or the question might be unanswerable. + +SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions +written adversarially by crowdworkers to look similar to answerable ones. +To do well on SQuAD2.0, systems must not only answer questions when possible, but also +determine when no answer is supported by the paragraph and abstain from answering. +""" + +_KWARGS_DESCRIPTION = """ +Computes SQuAD v2 scores (F1 and EM). +Args: + predictions: List of triple for question-answers to score with the following elements: + - the question-answer 'id' field as given in the references (see below) + - the text of the answer + - the probability that the question has no answer + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a list of Dict {'text': text of the answer as a string} + no_answer_threshold: float + Probability threshold to decide that a question has no answer. +Returns: + 'exact': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer + 'total': Number of score considered + 'HasAns_exact': Exact match (the normalized answer exactly match the gold answer) + 'HasAns_f1': The F-score of predicted tokens versus the gold answer + 'HasAns_total': Number of score considered + 'NoAns_exact': Exact match (the normalized answer exactly match the gold answer) + 'NoAns_f1': The F-score of predicted tokens versus the gold answer + 'NoAns_total': Number of score considered + 'best_exact': Best exact match (with varying threshold) + 'best_exact_thresh': No-answer probability threshold associated to the best exact match + 'best_f1': Best F1 (with varying threshold) + 'best_f1_thresh': No-answer probability threshold associated to the best F1 +Examples: + + >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] + >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] + >>> squad_v2_metric = evaluate.load("squad_v2") + >>> results = squad_v2_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class SquadV2(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "id": datasets.Value("string"), + "prediction_text": datasets.Value("string"), + "no_answer_probability": datasets.Value("float32"), + }, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + {"text": datasets.Value("string"), "answer_start": datasets.Value("int32")} + ), + }, + } + ), + codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + ) + + def _compute(self, predictions, references, no_answer_threshold=1.0): + no_answer_probabilities = {p["id"]: p["no_answer_probability"] for p in predictions} + dataset = [{"paragraphs": [{"qas": references}]}] + predictions = {p["id"]: p["prediction_text"] for p in predictions} + + qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + + exact_raw, f1_raw = get_raw_scores(dataset, predictions) + exact_thresh = apply_no_ans_threshold(exact_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) + f1_thresh = apply_no_ans_threshold(f1_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) + out_eval = make_eval_dict(exact_thresh, f1_thresh) + + if has_ans_qids: + has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) + merge_eval(out_eval, has_ans_eval, "HasAns") + if no_ans_qids: + no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + merge_eval(out_eval, no_ans_eval, "NoAns") + find_all_best_thresh(out_eval, predictions, exact_raw, f1_raw, no_answer_probabilities, qid_to_has_ans) + return dict(out_eval) diff --git a/ST/evaluate/metrics/super_glue/README.md b/ST/evaluate/metrics/super_glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..edcb4b8c8c224cbc3c041b96cf0ad4f1de897c1b --- /dev/null +++ b/ST/evaluate/metrics/super_glue/README.md @@ -0,0 +1,128 @@ +--- +title: SuperGLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. +--- + +# Metric Card for SuperGLUE + +## Metric description +This metric is used to compute the SuperGLUE evaluation metric associated to each of the subsets of the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). + +SuperGLUE is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. + + +## How to use + +There are two steps: (1) loading the SuperGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant SuperGLUE metric** : the subsets of SuperGLUE are the following: `boolq`, `cb`, `copa`, `multirc`, `record`, `rte`, `wic`, `wsc`, `wsc.fixed`, `axb`, `axg`. + +More information about the different subsets of the SuperGLUE dataset can be found on the [SuperGLUE dataset page](https://huggingface.co/datasets/super_glue) and on the [official dataset website](https://super.gluebenchmark.com/). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one list of reference labels. The structure of both inputs depends on the SuperGlUE subset being used: + +Format of `predictions`: +- for `record`: list of question-answer dictionaries with the following keys: + - `idx`: index of the question as specified by the dataset + - `prediction_text`: the predicted answer text +- for `multirc`: list of question-answer dictionaries with the following keys: + - `idx`: index of the question-answer pair as specified by the dataset + - `prediction`: the predicted answer label +- otherwise: list of predicted labels + +Format of `references`: +- for `record`: list of question-answers dictionaries with the following keys: + - `idx`: index of the question as specified by the dataset + - `answers`: list of possible answers +- otherwise: list of reference labels + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'copa') +predictions = [0, 1] +references = [0, 1] +results = super_glue_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the metric depends on the SuperGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`exact_match`: A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. (See [Exact Match](https://huggingface.co/metrics/exact_match) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. + +### Values from popular papers +The [original SuperGLUE paper](https://arxiv.org/pdf/1905.00537.pdf) reported average scores ranging from 47 to 71.5%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). + +For more recent model performance, see the [dataset leaderboard](https://super.gluebenchmark.com/leaderboard). + +## Examples + +Maximal values for the COPA subset (which outputs `accuracy`): + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] +predictions = [0, 1] +references = [0, 1] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0} +``` + +Minimal values for the MultiRC subset (which outputs `pearson` and `spearmanr`): + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'multirc') +predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] +references = [1,0] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 0.0, 'f1_m': 0.0, 'f1_a': 0.0} +``` + +Partial match for the COLA subset (which outputs `matthews_correlation`) + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'axb') +references = [0, 1] +predictions = [1,1] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'matthews_correlation': 0.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). + +The dataset also includes Winogender, a subset of the dataset that is designed to measure gender bias in coreference resolution systems. However, as noted in the SuperGLUE paper, this subset has its limitations: *"It offers only positive predictive value: A poor bias score is clear evidence that a model exhibits gender bias, but a good score does not mean that the model is unbiased.[...] Also, Winogender does not cover all forms of social bias, or even all forms of gender. For instance, the version of the data used here offers no coverage of gender-neutral they or non-binary pronouns." + +## Citation + +```bibtex +@article{wang2019superglue, + title={Super{GLUE}: A Stickier Benchmark for General-Purpose Language Understanding Systems}, + author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, + journal={arXiv preprint arXiv:1905.00537}, + year={2019} +} +``` + +## Further References + +- [SuperGLUE benchmark homepage](https://super.gluebenchmark.com/) diff --git a/ST/evaluate/metrics/super_glue/app.py b/ST/evaluate/metrics/super_glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0d11b1f706b3d99f16537dcec21c0a46b85b5bf4 --- /dev/null +++ b/ST/evaluate/metrics/super_glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("super_glue", "copa") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/super_glue/record_evaluation.py b/ST/evaluate/metrics/super_glue/record_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..396d7ac8abc6d52180fb8d960eddad9b03919a21 --- /dev/null +++ b/ST/evaluate/metrics/super_glue/record_evaluation.py @@ -0,0 +1,111 @@ +""" +Official evaluation script for ReCoRD v1.0. +(Some functions are adopted from the SQuAD evaluation script.) +""" + + +import argparse +import json +import re +import string +import sys +from collections import Counter + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1_score(prediction, ground_truth): + prediction_tokens = normalize_answer(prediction).split() + ground_truth_tokens = normalize_answer(ground_truth).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def evaluate(dataset, predictions): + f1 = exact_match = total = 0 + correct_ids = [] + for passage in dataset: + for qa in passage["qas"]: + total += 1 + if qa["id"] not in predictions: + message = f'Unanswered question {qa["id"]} will receive score 0.' + print(message, file=sys.stderr) + continue + + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + + _exact_match = metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + if int(_exact_match) == 1: + correct_ids.append(qa["id"]) + exact_match += _exact_match + + f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) + + exact_match = exact_match / total + f1 = f1 / total + + return {"exact_match": exact_match, "f1": f1}, correct_ids + + +if __name__ == "__main__": + expected_version = "1.0" + parser = argparse.ArgumentParser("Official evaluation script for ReCoRD v1.0.") + parser.add_argument("data_file", help="The dataset file in JSON format.") + parser.add_argument("pred_file", help="The model prediction file in JSON format.") + parser.add_argument("--output_correct_ids", action="store_true", help="Output the correctly answered query IDs.") + args = parser.parse_args() + + with open(args.data_file) as data_file: + dataset_json = json.load(data_file) + if dataset_json["version"] != expected_version: + print( + f'Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}', + file=sys.stderr, + ) + dataset = dataset_json["data"] + + with open(args.pred_file) as pred_file: + predictions = json.load(pred_file) + + metrics, correct_ids = evaluate(dataset, predictions) + + if args.output_correct_ids: + print(f"Output {len(correct_ids)} correctly answered question IDs.") + with open("correct_ids.json", "w") as f: + json.dump(correct_ids, f) diff --git a/ST/evaluate/metrics/super_glue/requirements.txt b/ST/evaluate/metrics/super_glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/super_glue/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/super_glue/super_glue.py b/ST/evaluate/metrics/super_glue/super_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..993a003b31c7f60536a42e2f4746f26382147ce9 --- /dev/null +++ b/ST/evaluate/metrics/super_glue/super_glue.py @@ -0,0 +1,237 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The SuperGLUE benchmark metric.""" + +import datasets +from sklearn.metrics import f1_score, matthews_corrcoef + +import evaluate + +from .record_evaluation import evaluate as evaluate_record + + +_CITATION = """\ +@article{wang2019superglue, + title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems}, + author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, + journal={arXiv preprint arXiv:1905.00537}, + year={2019} +} +""" + +_DESCRIPTION = """\ +SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after +GLUE with a new set of more difficult language understanding tasks, improved +resources, and a new public leaderboard. +""" + +_KWARGS_DESCRIPTION = """ +Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset. +Args: + predictions: list of predictions to score. Depending on the SuperGlUE subset: + - for 'record': list of question-answer dictionaries with the following keys: + - 'idx': index of the question as specified by the dataset + - 'prediction_text': the predicted answer text + - for 'multirc': list of question-answer dictionaries with the following keys: + - 'idx': index of the question-answer pair as specified by the dataset + - 'prediction': the predicted answer label + - otherwise: list of predicted labels + references: list of reference labels. Depending on the SuperGLUE subset: + - for 'record': list of question-answers dictionaries with the following keys: + - 'idx': index of the question as specified by the dataset + - 'answers': list of possible answers + - otherwise: list of reference labels +Returns: depending on the SuperGLUE subset: + - for 'record': + - 'exact_match': Exact match between answer and gold answer + - 'f1': F1 score + - for 'multirc': + - 'exact_match': Exact match between answer and gold answer + - 'f1_m': Per-question macro-F1 score + - 'f1_a': Average F1 score over all answers + - for 'axb': + 'matthews_correlation': Matthew Correlation + - for 'cb': + - 'accuracy': Accuracy + - 'f1': F1 score + - for all others: + - 'accuracy': Accuracy +Examples: + + >>> super_glue_metric = evaluate.load('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'cb') + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'record') + >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}] + >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 1.0, 'f1': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'multirc') + >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'axb') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'matthews_correlation': 1.0} +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels, f1_avg="binary"): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds, average=f1_avg)) + return { + "accuracy": acc, + "f1": f1, + } + + +def evaluate_multirc(ids_preds, labels): + """ + Computes F1 score and Exact Match for MultiRC predictions. + """ + question_map = {} + for id_pred, label in zip(ids_preds, labels): + question_id = f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}' + pred = id_pred["prediction"] + if question_id in question_map: + question_map[question_id].append((pred, label)) + else: + question_map[question_id] = [(pred, label)] + f1s, ems = [], [] + for question, preds_labels in question_map.items(): + question_preds, question_labels = zip(*preds_labels) + f1 = f1_score(y_true=question_labels, y_pred=question_preds, average="macro") + f1s.append(f1) + em = int(sum(p == l for p, l in preds_labels) == len(preds_labels)) + ems.append(em) + f1_m = float(sum(f1s) / len(f1s)) + em = sum(ems) / len(ems) + f1_a = float(f1_score(y_true=labels, y_pred=[id_pred["prediction"] for id_pred in ids_preds])) + return {"exact_match": em, "f1_m": f1_m, "f1_a": f1_a} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class SuperGlue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "boolq", + "cb", + "copa", + "multirc", + "record", + "rte", + "wic", + "wsc", + "wsc.fixed", + "axb", + "axg", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + codebase_urls=[], + reference_urls=[], + format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None, + ) + + def _get_feature_types(self): + if self.config_name == "record": + return { + "predictions": { + "idx": { + "passage": datasets.Value("int64"), + "query": datasets.Value("int64"), + }, + "prediction_text": datasets.Value("string"), + }, + "references": { + "idx": { + "passage": datasets.Value("int64"), + "query": datasets.Value("int64"), + }, + "answers": datasets.Sequence(datasets.Value("string")), + }, + } + elif self.config_name == "multirc": + return { + "predictions": { + "idx": { + "answer": datasets.Value("int64"), + "paragraph": datasets.Value("int64"), + "question": datasets.Value("int64"), + }, + "prediction": datasets.Value("int64"), + }, + "references": datasets.Value("int64"), + } + else: + return { + "predictions": datasets.Value("int64"), + "references": datasets.Value("int64"), + } + + def _compute(self, predictions, references): + if self.config_name == "axb": + return {"matthews_correlation": matthews_corrcoef(references, predictions)} + elif self.config_name == "cb": + return acc_and_f1(predictions, references, f1_avg="macro") + elif self.config_name == "record": + dataset = [ + { + "qas": [ + {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]} + for ref in references + ] + } + ] + predictions = {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions} + return evaluate_record(dataset, predictions)[0] + elif self.config_name == "multirc": + return evaluate_multirc(predictions, references) + elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' + ) diff --git a/ST/evaluate/metrics/ter/README.md b/ST/evaluate/metrics/ter/README.md new file mode 100644 index 0000000000000000000000000000000000000000..84aa2f948a00c2613002720a04abb183e2892a07 --- /dev/null +++ b/ST/evaluate/metrics/ter/README.md @@ -0,0 +1,175 @@ +--- +title: TER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a + hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu + (https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found + here: https://github.com/jhclark/tercom. + + The implementation here is slightly different from sacrebleu in terms of the required input format. The length of + the references and hypotheses lists need to be the same, so you may need to transpose your references compared to + sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + + See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. +--- + +# Metric Card for TER + +## Metric Description +TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a hypothesis requires to match a reference translation. We use the implementation that is already present in [sacrebleu](https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the [TERCOM implementation](https://github.com/jhclark/tercom). + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See [this github issue](https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534). + +See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. + + +## How to Use +This metric takes, at minimum, predicted sentences and reference sentences: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?", +... "What did the TER metric user say to the developer?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], +... ["Your jokes are...", "...TERrible"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... case_sensitive=True) +>>> print(results) +{'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} +``` + +### Inputs +This metric takes the following as input: +- **`predictions`** (`list` of `str`): The system stream (a sequence of segments). +- **`references`** (`list` of `list` of `str`): A list of one or more reference streams (each a sequence of segments). +- **`normalized`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. +- **`ignore_punct`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. +- **`support_zh_ja_chars`** (`boolean`): If `True`, tokenization/normalization supports processing of Chinese characters, as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. Only applies if `normalized = True`. Defaults to `False`. +- **`case_sensitive`** (`boolean`): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. + +### Output Values +This metric returns the following: +- **`score`** (`float`): TER score (num_edits / sum_ref_lengths * 100) +- **`num_edits`** (`int`): The cumulative number of edits +- **`ref_length`** (`float`): The cumulative average reference length + +The output takes the following form: +```python +{'score': ter_score, 'num_edits': num_edits, 'ref_length': ref_length} +``` + +The metric can take on any value `0` and above. `0` is a perfect score, meaning the predictions exactly match the references and no edits were necessary. Higher scores are worse. Scores above 100 mean that the cumulative number of edits, `num_edits`, is higher than the cumulative length of the references, `ref_length`. + +#### Values from Popular Papers + + +### Examples +Basic example with only predictions and references as inputs: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... case_sensitive=True) +>>> print(results) +{'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} +``` + +Example with `normalization = True`: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... normalized=True, +... case_sensitive=True) +>>> print(results) +{'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} +``` + +Example ignoring punctuation and capitalization, and everything matches: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... ignore_punct=True, +... case_sensitive=False) +>>> print(results) +{'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} +``` + +Example ignoring punctuation and capitalization, but with an extra (incorrect) sample: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?", +... "What did the TER metric user say to the developer?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], +... ["Your jokes are...", "...TERrible"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... ignore_punct=True, +... case_sensitive=False) +>>> print(results) +{'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} +``` + + +## Limitations and Bias + + +## Citation +```bibtex +@inproceedings{snover-etal-2006-study, + title = "A Study of Translation Edit Rate with Targeted Human Annotation", + author = "Snover, Matthew and + Dorr, Bonnie and + Schwartz, Rich and + Micciulla, Linnea and + Makhoul, John", + booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", + month = aug # " 8-12", + year = "2006", + address = "Cambridge, Massachusetts, USA", + publisher = "Association for Machine Translation in the Americas", + url = "https://aclanthology.org/2006.amta-papers.25", + pages = "223--231", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See [the sacreBLEU github repo](https://github.com/mjpost/sacreBLEU#ter) for more information. diff --git a/ST/evaluate/metrics/ter/app.py b/ST/evaluate/metrics/ter/app.py new file mode 100644 index 0000000000000000000000000000000000000000..063222ebb9814a282c1b479b84e58adc8bd3bf0e --- /dev/null +++ b/ST/evaluate/metrics/ter/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("ter") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/ter/requirements.txt b/ST/evaluate/metrics/ter/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/ST/evaluate/metrics/ter/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/ST/evaluate/metrics/ter/ter.py b/ST/evaluate/metrics/ter/ter.py new file mode 100644 index 0000000000000000000000000000000000000000..4adb9986cc4c5cd08fe03246d4dc8f54a31e0839 --- /dev/null +++ b/ST/evaluate/metrics/ter/ter.py @@ -0,0 +1,212 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TER metric as available in sacrebleu. """ +import datasets +import sacrebleu as scb +from packaging import version +from sacrebleu import TER + +import evaluate + + +_CITATION = """\ +@inproceedings{snover-etal-2006-study, + title = "A Study of Translation Edit Rate with Targeted Human Annotation", + author = "Snover, Matthew and + Dorr, Bonnie and + Schwartz, Rich and + Micciulla, Linnea and + Makhoul, John", + booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", + month = aug # " 8-12", + year = "2006", + address = "Cambridge, Massachusetts, USA", + publisher = "Association for Machine Translation in the Americas", + url = "https://aclanthology.org/2006.amta-papers.25", + pages = "223--231", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a +hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu +(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found +here: https://github.com/jhclark/tercom. + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of +the references and hypotheses lists need to be the same, so you may need to transpose your references compared to +sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces TER scores alongside the number of edits and reference length. + +Args: + predictions (list of str): The system stream (a sequence of segments). + references (list of list of str): A list of one or more reference streams (each a sequence of segments). + normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. + ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. + support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters, + as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. + Only applies if `normalized = True`. Defaults to `False`. + case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. + +Returns: + 'score' (float): TER score (num_edits / sum_ref_lengths * 100) + 'num_edits' (int): The cumulative number of edits + 'ref_length' (float): The cumulative average reference length + +Examples: + Example 1: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?", + ... "What did the TER metric user say to the developer?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], + ... ["Your jokes are...", "...TERrible"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... case_sensitive=True) + >>> print(results) + {'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} + + Example 2: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... case_sensitive=True) + >>> print(results) + {'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} + + Example 3: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... normalized=True, + ... case_sensitive=True) + >>> print(results) + {'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} + + Example 4: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... ignore_punct=True, + ... case_sensitive=False) + >>> print(results) + {'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} + + Example 5: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?", + ... "What did the TER metric user say to the developer?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], + ... ["Your jokes are...", "...TERrible"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... ignore_punct=True, + ... case_sensitive=False) + >>> print(results) + {'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Ter(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="http://www.cs.umd.edu/~snover/tercom/", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU#ter"], + reference_urls=[ + "https://github.com/jhclark/tercom", + ], + ) + + def _compute( + self, + predictions, + references, + normalized: bool = False, + ignore_punct: bool = False, + support_zh_ja_chars: bool = False, + case_sensitive: bool = False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + + sb_ter = TER( + normalized=normalized, + no_punct=ignore_punct, + asian_support=support_zh_ja_chars, + case_sensitive=case_sensitive, + ) + output = sb_ter.corpus_score(predictions, transformed_references) + + return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length} diff --git a/ST/evaluate/metrics/trec_eval/README.md b/ST/evaluate/metrics/trec_eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7512eb55ee41469e83db4a637bc9c61b6afe4e41 --- /dev/null +++ b/ST/evaluate/metrics/trec_eval/README.md @@ -0,0 +1,149 @@ +--- +title: TREC Eval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The TREC Eval metric combines a number of information retrieval metrics such as precision and nDCG. It is used to score rankings of retrieved documents with reference values. +--- + +# Metric Card for TREC Eval + +## Metric Description + +The TREC Eval metric combines a number of information retrieval metrics such as precision and normalized Discounted Cumulative Gain (nDCG). It is used to score rankings of retrieved documents with reference values. + +## How to Use +```Python +from evaluate import load +trec_eval = load("trec_eval") +results = trec_eval.compute(predictions=[run], references=[qrel]) +``` + +### Inputs +- **predictions** *(dict): a single retrieval run.* + - **query** *(int): Query ID.* + - **q0** *(str): Literal `"q0"`.* + - **docid** *(str): Document ID.* + - **rank** *(int): Rank of document.* + - **score** *(float): Score of document.* + - **system** *(str): Tag for current run.* +- **references** *(dict): a single qrel.* + - **query** *(int): Query ID.* + - **q0** *(str): Literal `"q0"`.* + - **docid** *(str): Document ID.* + - **rel** *(int): Relevance of document.* + +### Output Values +- **runid** *(str): Run name.* +- **num_ret** *(int): Number of retrieved documents.* +- **num_rel** *(int): Number of relevant documents.* +- **num_rel_ret** *(int): Number of retrieved relevant documents.* +- **num_q** *(int): Number of queries.* +- **map** *(float): Mean average precision.* +- **gm_map** *(float): geometric mean average precision.* +- **bpref** *(float): binary preference score.* +- **Rprec** *(float): precision@R, where R is number of relevant documents.* +- **recip_rank** *(float): reciprocal rank* +- **P@k** *(float): precision@k (k in [5, 10, 15, 20, 30, 100, 200, 500, 1000]).* +- **NDCG@k** *(float): nDCG@k (k in [5, 10, 15, 20, 30, 100, 200, 500, 1000]).* + +### Examples + +A minimal example of looks as follows: +```Python +qrel = { + "query": [0], + "q0": ["q0"], + "docid": ["doc_1"], + "rel": [2] +} +run = { + "query": [0, 0], + "q0": ["q0", "q0"], + "docid": ["doc_2", "doc_1"], + "rank": [0, 1], + "score": [1.5, 1.2], + "system": ["test", "test"] +} + +trec_eval = evaluate.load("trec_eval") +results = trec_eval.compute(references=[qrel], predictions=[run]) +results["P@5"] +0.2 +``` + +A more realistic use case with an examples from [`trectools`](https://github.com/joaopalotti/trectools): + +```python +qrel = pd.read_csv("robust03_qrels.txt", sep="\s+", names=["query", "q0", "docid", "rel"]) +qrel["q0"] = qrel["q0"].astype(str) +qrel = qrel.to_dict(orient="list") + +run = pd.read_csv("input.InexpC2", sep="\s+", names=["query", "q0", "docid", "rank", "score", "system"]) +run = run.to_dict(orient="list") + +trec_eval = evaluate.load("trec_eval") +result = trec_eval.compute(run=[run], qrel=[qrel]) +``` + +```python +result + +{'runid': 'InexpC2', + 'num_ret': 100000, + 'num_rel': 6074, + 'num_rel_ret': 3198, + 'num_q': 100, + 'map': 0.22485930431817494, + 'gm_map': 0.10411523825735523, + 'bpref': 0.217511695914079, + 'Rprec': 0.2502547201167236, + 'recip_rank': 0.6646545943335417, + 'P@5': 0.44, + 'P@10': 0.37, + 'P@15': 0.34600000000000003, + 'P@20': 0.30999999999999994, + 'P@30': 0.2563333333333333, + 'P@100': 0.1428, + 'P@200': 0.09510000000000002, + 'P@500': 0.05242, + 'P@1000': 0.03198, + 'NDCG@5': 0.4101480395089769, + 'NDCG@10': 0.3806761417784469, + 'NDCG@15': 0.37819463408955706, + 'NDCG@20': 0.3686080836061317, + 'NDCG@30': 0.352474353427451, + 'NDCG@100': 0.3778329431025776, + 'NDCG@200': 0.4119129817248979, + 'NDCG@500': 0.4585354576461375, + 'NDCG@1000': 0.49092149290805653} +``` + +## Limitations and Bias +The `trec_eval` metric requires the inputs to be in the TREC run and qrel formats for predictions and references. + + +## Citation + +```bibtex +@inproceedings{palotti2019, + author = {Palotti, Joao and Scells, Harrisen and Zuccon, Guido}, + title = {TrecTools: an open-source Python library for Information Retrieval practitioners involved in TREC-like campaigns}, + series = {SIGIR'19}, + year = {2019}, + location = {Paris, France}, + publisher = {ACM} +} +``` + +## Further References + +- Homepage: https://github.com/joaopalotti/trectools \ No newline at end of file diff --git a/ST/evaluate/metrics/trec_eval/app.py b/ST/evaluate/metrics/trec_eval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a15c15c2439a4a063da49e3c32745574ada77fea --- /dev/null +++ b/ST/evaluate/metrics/trec_eval/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("trec_eval") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/trec_eval/requirements.txt b/ST/evaluate/metrics/trec_eval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a8124560867182142d4cb3c0d94b0646105080c --- /dev/null +++ b/ST/evaluate/metrics/trec_eval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +trectools \ No newline at end of file diff --git a/ST/evaluate/metrics/trec_eval/trec_eval.py b/ST/evaluate/metrics/trec_eval/trec_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..462374e01a1972146dc87c0fa70864711046ee2e --- /dev/null +++ b/ST/evaluate/metrics/trec_eval/trec_eval.py @@ -0,0 +1,139 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module to compute TREC evaluation scores.""" + +import datasets +import pandas as pd +from trectools import TrecEval, TrecQrel, TrecRun + +import evaluate + + +_CITATION = """\ +@inproceedings{palotti2019, + author = {Palotti, Joao and Scells, Harrisen and Zuccon, Guido}, + title = {TrecTools: an open-source Python library for Information Retrieval practitioners involved in TREC-like campaigns}, + series = {SIGIR'19}, + year = {2019}, + location = {Paris, France}, + publisher = {ACM} +} +""" + +# TODO: Add description of the module here +_DESCRIPTION = """\ +The TREC Eval metric combines a number of information retrieval metrics such as \ +precision and nDCG. It is used to score rankings of retrieved documents with reference values.""" + + +# TODO: Add description of the arguments of the module here +_KWARGS_DESCRIPTION = """ +Calculates TREC evaluation scores based on a run and qrel. +Args: + predictions: list containing a single run. + references: list containing a single qrel. +Returns: + dict: TREC evaluation scores. +Examples: + >>> trec = evaluate.load("trec_eval") + >>> qrel = { + ... "query": [0], + ... "q0": ["0"], + ... "docid": ["doc_1"], + ... "rel": [2] + ... } + >>> run = { + ... "query": [0, 0], + ... "q0": ["q0", "q0"], + ... "docid": ["doc_2", "doc_1"], + ... "rank": [0, 1], + ... "score": [1.5, 1.2], + ... "system": ["test", "test"] + ... } + >>> results = trec.compute(references=[qrel], predictions=[run]) + >>> print(results["P@5"]) + 0.2 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class TRECEval(evaluate.Metric): + """Compute TREC evaluation scores.""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "query": datasets.Sequence(datasets.Value("int64")), + "q0": datasets.Sequence(datasets.Value("string")), + "docid": datasets.Sequence(datasets.Value("string")), + "rank": datasets.Sequence(datasets.Value("int64")), + "score": datasets.Sequence(datasets.Value("float")), + "system": datasets.Sequence(datasets.Value("string")), + }, + "references": { + "query": datasets.Sequence(datasets.Value("int64")), + "q0": datasets.Sequence(datasets.Value("string")), + "docid": datasets.Sequence(datasets.Value("string")), + "rel": datasets.Sequence(datasets.Value("int64")), + }, + } + ), + homepage="https://github.com/joaopalotti/trectools", + ) + + def _compute(self, references, predictions): + """Returns the TREC evaluation scores.""" + + if len(predictions) > 1 or len(references) > 1: + raise ValueError( + f"You can only pass one prediction and reference per evaluation. You passed {len(predictions)} prediction(s) and {len(references)} reference(s)." + ) + + df_run = pd.DataFrame(predictions[0]) + df_qrel = pd.DataFrame(references[0]) + + trec_run = TrecRun() + trec_run.filename = "placeholder.file" + trec_run.run_data = df_run + + trec_qrel = TrecQrel() + trec_qrel.filename = "placeholder.file" + trec_qrel.qrels_data = df_qrel + + trec_eval = TrecEval(trec_run, trec_qrel) + + result = {} + result["runid"] = trec_eval.run.get_runid() + result["num_ret"] = trec_eval.get_retrieved_documents(per_query=False) + result["num_rel"] = trec_eval.get_relevant_documents(per_query=False) + result["num_rel_ret"] = trec_eval.get_relevant_retrieved_documents(per_query=False) + result["num_q"] = len(trec_eval.run.topics()) + result["map"] = trec_eval.get_map(depth=10000, per_query=False, trec_eval=True) + result["gm_map"] = trec_eval.get_geometric_map(depth=10000, trec_eval=True) + result["bpref"] = trec_eval.get_bpref(depth=1000, per_query=False, trec_eval=True) + result["Rprec"] = trec_eval.get_rprec(depth=1000, per_query=False, trec_eval=True) + result["recip_rank"] = trec_eval.get_reciprocal_rank(depth=1000, per_query=False, trec_eval=True) + + for v in [5, 10, 15, 20, 30, 100, 200, 500, 1000]: + result[f"P@{v}"] = trec_eval.get_precision(depth=v, per_query=False, trec_eval=True) + for v in [5, 10, 15, 20, 30, 100, 200, 500, 1000]: + result[f"NDCG@{v}"] = trec_eval.get_ndcg(depth=v, per_query=False, trec_eval=True) + + return result diff --git a/ST/evaluate/metrics/wer/README.md b/ST/evaluate/metrics/wer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21f83429fce2e58cf36c210d057de761640b2281 --- /dev/null +++ b/ST/evaluate/metrics/wer/README.md @@ -0,0 +1,158 @@ +--- +title: WER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. + + The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + + This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate. + + Word error rate can then be computed as: + + WER = (S + D + I) / N = (S + D + I) / (S + D + C) + + where + + S is the number of substitutions, + D is the number of deletions, + I is the number of insertions, + C is the number of correct words, + N is the number of words in the reference (N=S+D+C). + + This value indicates the average number of errors per reference word. The lower the value, the better the + performance of the ASR system with a WER of 0 being a perfect score. +--- + +# Metric Card for WER + +## Metric description +Word error rate (WER) is a common metric of the performance of an automatic speech recognition (ASR) system. + +The general difficulty of measuring the performance of ASR systems lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance), working at the word level. + +This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between [perplexity](https://huggingface.co/metrics/perplexity) and word error rate (see [this article](https://www.cs.cmu.edu/~roni/papers/eval-metrics-bntuw-9802.pdf) for further information). + +Word error rate can then be computed as: + +`WER = (S + D + I) / N = (S + D + I) / (S + D + C)` + +where + +`S` is the number of substitutions, + +`D` is the number of deletions, + +`I` is the number of insertions, + +`C` is the number of correct words, + +`N` is the number of words in the reference (`N=S+D+C`). + + +## How to use + +The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). + + +```python +from evaluate import load +wer = load("wer") +wer_score = wer.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a float representing the word error rate. + +``` +print(wer_score) +0.5 +``` + +This value indicates the average number of errors per reference word. + +The **lower** the value, the **better** the performance of the ASR system, with a WER of 0 being a perfect score. + +### Values from popular papers + +This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. + +For example, datasets such as [LibriSpeech](https://huggingface.co/datasets/librispeech_asr) report a WER in the 1.8-3.3 range, whereas ASR models evaluated on [Timit](https://huggingface.co/datasets/timit_asr) report a WER in the 8.3-20.4 range. +See the leaderboards for [LibriSpeech](https://paperswithcode.com/sota/speech-recognition-on-librispeech-test-clean) and [Timit](https://paperswithcode.com/sota/speech-recognition-on-timit) for the most recent values. + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["hello world", "good night moon"] +references = ["hello world", "good night moon"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +0.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["this is the prediction", "there is an other sample"] +references = ["this is the reference", "there is another one"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +0.5 +``` + +No match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["hello world", "good night moon"] +references = ["hi everyone", "have a great day"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +1.0 +``` + +## Limitations and bias + +WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +## Citation + +```bibtex +@inproceedings{woodard1982, +author = {Woodard, J.P. and Nelson, J.T., +year = {1982}, +journal = {Workshop on standardisation for speech I/O technology, Naval Air Development Center, Warminster, PA}, +title = {An information theoretic measure of speech recognition performance} +} +``` + +```bibtex +@inproceedings{morris2004, +author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, +year = {2004}, +month = {01}, +pages = {}, +title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +``` + +## Further References + +- [Word Error Rate -- Wikipedia](https://en.wikipedia.org/wiki/Word_error_rate) +- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/ST/evaluate/metrics/wer/app.py b/ST/evaluate/metrics/wer/app.py new file mode 100644 index 0000000000000000000000000000000000000000..30b8fe447f13fd199a4d6388fb689450975db7b7 --- /dev/null +++ b/ST/evaluate/metrics/wer/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wer") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/wer/requirements.txt b/ST/evaluate/metrics/wer/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..aeeda9ad86ac3257a56a69d358c7d863236ac87f --- /dev/null +++ b/ST/evaluate/metrics/wer/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +jiwer \ No newline at end of file diff --git a/ST/evaluate/metrics/wer/wer.py b/ST/evaluate/metrics/wer/wer.py new file mode 100644 index 0000000000000000000000000000000000000000..214d5b22e2afbe6bf7d78747dd3f00b9c76b8a3e --- /dev/null +++ b/ST/evaluate/metrics/wer/wer.py @@ -0,0 +1,106 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Word Error Ratio (WER) metric. """ + +import datasets +from jiwer import compute_measures + +import evaluate + + +_CITATION = """\ +@inproceedings{inproceedings, + author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, + year = {2004}, + month = {01}, + pages = {}, + title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +""" + +_DESCRIPTION = """\ +Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. + +The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate. + +Word error rate can then be computed as: + +WER = (S + D + I) / N = (S + D + I) / (S + D + C) + +where + +S is the number of substitutions, +D is the number of deletions, +I is the number of insertions, +C is the number of correct words, +N is the number of words in the reference (N=S+D+C). + +This value indicates the average number of errors per reference word. The lower the value, the better the +performance of the ASR system with a WER of 0 being a perfect score. +""" + +_KWARGS_DESCRIPTION = """ +Compute WER score of transcribed segments against references. + +Args: + references: List of references for each speech input. + predictions: List of transcriptions to score. + concatenate_texts (bool, default=False): Whether to concatenate all input texts or compute WER iteratively. + +Returns: + (float): the word error rate + +Examples: + + >>> predictions = ["this is the prediction", "there is an other sample"] + >>> references = ["this is the reference", "there is another one"] + >>> wer = evaluate.load("wer") + >>> wer_score = wer.compute(predictions=predictions, references=references) + >>> print(wer_score) + 0.5 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WER(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/jitsi/jiwer/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Word_error_rate", + ], + ) + + def _compute(self, predictions=None, references=None, concatenate_texts=False): + if concatenate_texts: + return compute_measures(references, predictions)["wer"] + else: + incorrect = 0 + total = 0 + for prediction, reference in zip(predictions, references): + measures = compute_measures(reference, prediction) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + return incorrect / total diff --git a/ST/evaluate/metrics/wiki_split/README.md b/ST/evaluate/metrics/wiki_split/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6736bcc7d4b33c4772d4243a9fa05986598f4bb1 --- /dev/null +++ b/ST/evaluate/metrics/wiki_split/README.md @@ -0,0 +1,120 @@ +--- +title: WikiSplit +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU + It can be used to evaluate the quality of machine-generated texts. +--- + +# Metric Card for WikiSplit + +## Metric description + +WikiSplit is the combination of three metrics: [SARI](https://huggingface.co/metrics/sari), [exact match](https://huggingface.co/metrics/exact_match) and [SacreBLEU](https://huggingface.co/metrics/sacrebleu). + +It can be used to evaluate the quality of sentence splitting approaches, which require rewriting a long sentence into two or more coherent short sentences, e.g. based on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). + +## How to use + +The WIKI_SPLIT metric takes three inputs: + +`sources`: a list of source sentences, where each sentence should be a string. + +`predictions`: a list of predicted sentences, where each sentence should be a string. + +`references`: a list of lists of reference sentences, where each sentence should be a string. + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 you now get in ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary containing three scores: + +`sari`: the [SARI](https://huggingface.co/metrics/sari) score, whose range is between `0.0` and `100.0` -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. + +`sacrebleu`: the [SacreBLEU](https://huggingface.co/metrics/sacrebleu) score, which can take any value between `0.0` and `100.0`, inclusive. + +`exact`: the [exact match](https://huggingface.co/metrics/exact_match) score, which represents the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. It ranges from `0.0` to `100`, inclusive. Here, `0.0` means no prediction/reference pairs were matches, while `100.0` means they all were. + +```python +>>> print(results) +{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +``` + +### Values from popular papers + +This metric was initially used by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf) to evaluate the performance of different split-and-rephrase approaches on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). They reported a SARI score of 63.5, a SacreBLEU score of 77.2, and an EXACT_MATCH score of 16.3. + +## Examples + +Perfect match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 species are currently accepted ."] +>>> references= [["About 95 species are currently accepted ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 100.0, 'sacrebleu': 100.00000000000004, 'exact': 100.0 +``` + +Partial match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 you now get in ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +``` + +No match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["Hello world ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 14.047619047619046, 'sacrebleu': 0.0, 'exact': 0.0} +``` +## Limitations and bias + +This metric is not the official metric to evaluate models on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). It was initially proposed by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf), whereas the [original paper introducing the WikiSplit dataset (2018)](https://aclanthology.org/D18-1080.pdf) uses different metrics to evaluate performance, such as corpus-level [BLEU](https://huggingface.co/metrics/bleu) and sentence-level BLEU. + +## Citation + +```bibtex +@article{rothe2020leveraging, + title={Leveraging pre-trained checkpoints for sequence generation tasks}, + author={Rothe, Sascha and Narayan, Shashi and Severyn, Aliaksei}, + journal={Transactions of the Association for Computational Linguistics}, + volume={8}, + pages={264--280}, + year={2020}, + publisher={MIT Press} +} +``` + +## Further References + +- [WikiSplit dataset](https://huggingface.co/datasets/wiki_split) +- [WikiSplit paper (Botha et al., 2018)](https://aclanthology.org/D18-1080.pdf) diff --git a/ST/evaluate/metrics/wiki_split/app.py b/ST/evaluate/metrics/wiki_split/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6e077faacdec6e69fa5d9c696350847ccce311c4 --- /dev/null +++ b/ST/evaluate/metrics/wiki_split/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wiki_split") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/wiki_split/requirements.txt b/ST/evaluate/metrics/wiki_split/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a85cf386b67ac245901a4fe6a22bf7edf6143b1 --- /dev/null +++ b/ST/evaluate/metrics/wiki_split/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu +sacremoses \ No newline at end of file diff --git a/ST/evaluate/metrics/wiki_split/wiki_split.py b/ST/evaluate/metrics/wiki_split/wiki_split.py new file mode 100644 index 0000000000000000000000000000000000000000..be83681b741e8767b58ae3924aaf51b78f8e314b --- /dev/null +++ b/ST/evaluate/metrics/wiki_split/wiki_split.py @@ -0,0 +1,366 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" WIKI_SPLIT metric.""" + +import re +import string +from collections import Counter + +import datasets +import sacrebleu +import sacremoses +from packaging import version + +import evaluate + + +_CITATION = """ +@inproceedings{xu-etal-2016-optimizing, + title = {Optimizing Statistical Machine Translation for Text Simplification}, + authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {4}, + year={2016}, + url = {https://www.aclweb.org/anthology/Q16-1029}, + pages = {401--415 +}, +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU +It can be used to evaluate the quality of machine-generated texts. +""" + + +_KWARGS_DESCRIPTION = """ +Calculates sari score (between 0 and 100) given a list of source and predicted +sentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score. +Args: + sources: list of source sentences where each sentence should be a string. + predictions: list of predicted sentences where each sentence should be a string. + references: list of lists of reference sentences where each sentence should be a string. +Returns: + sari: sari score + sacrebleu: sacrebleu score + exact: exact score + +Examples: + >>> sources=["About 95 species are currently accepted ."] + >>> predictions=["About 95 you now get in ."] + >>> references=[["About 95 species are currently known ."]] + >>> wiki_split = evaluate.load("wiki_split") + >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) + >>> print(results) + {'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +""" + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_em(predictions, references): + scores = [any([compute_exact(ref, pred) for ref in refs]) for pred, refs in zip(predictions, references)] + return (sum(scores) / len(scores)) * 100 + + +def SARIngram(sgrams, cgrams, rgramslist, numref): + rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] + rgramcounter = Counter(rgramsall) + + sgramcounter = Counter(sgrams) + sgramcounter_rep = Counter() + for sgram, scount in sgramcounter.items(): + sgramcounter_rep[sgram] = scount * numref + + cgramcounter = Counter(cgrams) + cgramcounter_rep = Counter() + for cgram, ccount in cgramcounter.items(): + cgramcounter_rep[cgram] = ccount * numref + + # KEEP + keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep + keepgramcountergood_rep = keepgramcounter_rep & rgramcounter + keepgramcounterall_rep = sgramcounter_rep & rgramcounter + + keeptmpscore1 = 0 + keeptmpscore2 = 0 + for keepgram in keepgramcountergood_rep: + keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] + # Fix an alleged bug [2] in the keep score computation. + # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] + keeptmpscore2 += keepgramcountergood_rep[keepgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + keepscore_precision = 1 + keepscore_recall = 1 + if len(keepgramcounter_rep) > 0: + keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) + if len(keepgramcounterall_rep) > 0: + # Fix an alleged bug [2] in the keep score computation. + # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) + keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) + keepscore = 0 + if keepscore_precision > 0 or keepscore_recall > 0: + keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) + + # DELETION + delgramcounter_rep = sgramcounter_rep - cgramcounter_rep + delgramcountergood_rep = delgramcounter_rep - rgramcounter + delgramcounterall_rep = sgramcounter_rep - rgramcounter + deltmpscore1 = 0 + deltmpscore2 = 0 + for delgram in delgramcountergood_rep: + deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] + deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + delscore_precision = 1 + if len(delgramcounter_rep) > 0: + delscore_precision = deltmpscore1 / len(delgramcounter_rep) + + # ADDITION + addgramcounter = set(cgramcounter) - set(sgramcounter) + addgramcountergood = set(addgramcounter) & set(rgramcounter) + addgramcounterall = set(rgramcounter) - set(sgramcounter) + + addtmpscore = 0 + for addgram in addgramcountergood: + addtmpscore += 1 + + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + addscore_precision = 1 + addscore_recall = 1 + if len(addgramcounter) > 0: + addscore_precision = addtmpscore / len(addgramcounter) + if len(addgramcounterall) > 0: + addscore_recall = addtmpscore / len(addgramcounterall) + addscore = 0 + if addscore_precision > 0 or addscore_recall > 0: + addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) + + return (keepscore, delscore_precision, addscore) + + +def SARIsent(ssent, csent, rsents): + numref = len(rsents) + + s1grams = ssent.split(" ") + c1grams = csent.split(" ") + s2grams = [] + c2grams = [] + s3grams = [] + c3grams = [] + s4grams = [] + c4grams = [] + + r1gramslist = [] + r2gramslist = [] + r3gramslist = [] + r4gramslist = [] + for rsent in rsents: + r1grams = rsent.split(" ") + r2grams = [] + r3grams = [] + r4grams = [] + r1gramslist.append(r1grams) + for i in range(0, len(r1grams) - 1): + if i < len(r1grams) - 1: + r2gram = r1grams[i] + " " + r1grams[i + 1] + r2grams.append(r2gram) + if i < len(r1grams) - 2: + r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + r3grams.append(r3gram) + if i < len(r1grams) - 3: + r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] + r4grams.append(r4gram) + r2gramslist.append(r2grams) + r3gramslist.append(r3grams) + r4gramslist.append(r4grams) + + for i in range(0, len(s1grams) - 1): + if i < len(s1grams) - 1: + s2gram = s1grams[i] + " " + s1grams[i + 1] + s2grams.append(s2gram) + if i < len(s1grams) - 2: + s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + s3grams.append(s3gram) + if i < len(s1grams) - 3: + s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] + s4grams.append(s4gram) + + for i in range(0, len(c1grams) - 1): + if i < len(c1grams) - 1: + c2gram = c1grams[i] + " " + c1grams[i + 1] + c2grams.append(c2gram) + if i < len(c1grams) - 2: + c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + c3grams.append(c3gram) + if i < len(c1grams) - 3: + c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] + c4grams.append(c4gram) + + (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) + (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) + (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) + (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) + avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 + avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 + avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 + finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 + return finalscore + + +def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): + + # Normalization is requried for the ASSET dataset (one of the primary + # datasets in sentence simplification) to allow using space + # to split the sentence. Even though Wiki-Auto and TURK datasets, + # do not require normalization, we do it for consistency. + # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. + # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 + + if lowercase: + sentence = sentence.lower() + + if tokenizer in ["13a", "intl"]: + if version.parse(sacrebleu.__version__).major >= 2: + normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) + else: + normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) + elif tokenizer == "moses": + normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) + elif tokenizer == "penn": + normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) + else: + normalized_sent = sentence + + if not return_str: + normalized_sent = normalized_sent.split() + + return normalized_sent + + +def compute_sari(sources, predictions, references): + + if not (len(sources) == len(predictions) == len(references)): + raise ValueError("Sources length must match predictions and references lengths.") + sari_score = 0 + for src, pred, refs in zip(sources, predictions, references): + sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) + sari_score = sari_score / len(predictions) + return 100 * sari_score + + +def compute_sacrebleu( + predictions, + references, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + use_effective_order=False, +): + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + output = sacrebleu.corpus_bleu( + predictions, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + ) + return output.score + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WikiSplit(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=[ + "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py", + "https://github.com/cocoxu/simplification/blob/master/SARI.py", + "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", + "https://github.com/mjpost/sacreBLEU", + ], + reference_urls=[ + "https://www.aclweb.org/anthology/Q16-1029.pdf", + "https://github.com/mjpost/sacreBLEU", + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute(self, sources, predictions, references): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + result = {} + result.update({"sari": compute_sari(sources=sources, predictions=predictions, references=references)}) + result.update({"sacrebleu": compute_sacrebleu(predictions=predictions, references=references)}) + result.update({"exact": compute_em(predictions=predictions, references=references)}) + return result diff --git a/ST/evaluate/metrics/xnli/README.md b/ST/evaluate/metrics/xnli/README.md new file mode 100644 index 0000000000000000000000000000000000000000..beb4066f40b440470f67e29ab2f5908820d234eb --- /dev/null +++ b/ST/evaluate/metrics/xnli/README.md @@ -0,0 +1,119 @@ +--- +title: XNLI +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + XNLI is a subset of a few thousand examples from MNLI which has been translated + into a 14 different languages (some low-ish resource). As with MNLI, the goal is + to predict textual entailment (does sentence A imply/contradict/neither sentence + B) and is a classification task (given two sentences, predict one of three + labels). +--- + +# Metric Card for XNLI + +## Metric description + +The XNLI metric allows to evaluate a model's score on the [XNLI dataset](https://huggingface.co/datasets/xnli), which is a subset of a few thousand examples from the [MNLI dataset](https://huggingface.co/datasets/glue/viewer/mnli) that have been translated into a 14 different languages, some of which are relatively low resource such as Swahili and Urdu. + +As with MNLI, the task is to predict textual entailment (does sentence A imply/contradict/neither sentence B) and is a classification task (given two sentences, predict one of three labels). + +## How to use + +The XNLI metric is computed based on the `predictions` (a list of predicted labels) and the `references` (a list of ground truth labels). + +```python +from evaluate import load +xnli_metric = load("xnli") +predictions = [0, 1] +references = [0, 1] +results = xnli_metric.compute(predictions=predictions, references=references) +``` + +## Output values + +The output of the XNLI metric is simply the `accuracy`, i.e. the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +### Values from popular papers +The [original XNLI paper](https://arxiv.org/pdf/1809.05053.pdf) reported accuracies ranging from 59.3 (for `ur`) to 73.7 (for `en`) for the BiLSTM-max model. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/xnli). + +## Examples + +Maximal values: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [0, 1] +>>> references = [0, 1] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 1.0} +``` + +Minimal values: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [1, 0] +>>> references = [0, 1] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 0.0} +``` + +Partial match: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [1, 0, 1] +>>> references = [1, 0, 0] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 0.6666666666666666} +``` + +## Limitations and bias + +While accuracy alone does give a certain indication of performance, it can be supplemented by error analysis and a better understanding of the model's mistakes on each of the categories represented in the dataset, especially if they are unbalanced. + +While the XNLI dataset is multilingual and represents a diversity of languages, in reality, cross-lingual sentence understanding goes beyond translation, given that there are many cultural differences that have an impact on human sentiment annotations. Since the XNLI dataset was obtained by translation based on English sentences, it does not capture these cultural differences. + + + +## Citation + +```bibtex +@InProceedings{conneau2018xnli, + author = "Conneau, Alexis + and Rinott, Ruty + and Lample, Guillaume + and Williams, Adina + and Bowman, Samuel R. + and Schwenk, Holger + and Stoyanov, Veselin", + title = "XNLI: Evaluating Cross-lingual Sentence Representations", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods + in Natural Language Processing", + year = "2018", + publisher = "Association for Computational Linguistics", + location = "Brussels, Belgium", +} +``` + +## Further References + +- [XNI Dataset GitHub](https://github.com/facebookresearch/XNLI) +- [HuggingFace Tasks -- Text Classification](https://huggingface.co/tasks/text-classification) diff --git a/ST/evaluate/metrics/xnli/app.py b/ST/evaluate/metrics/xnli/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a44a7ae86b81a933d2b90ffab37d28a11ecc2d99 --- /dev/null +++ b/ST/evaluate/metrics/xnli/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("xnli") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/xnli/requirements.txt b/ST/evaluate/metrics/xnli/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/ST/evaluate/metrics/xnli/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/ST/evaluate/metrics/xnli/xnli.py b/ST/evaluate/metrics/xnli/xnli.py new file mode 100644 index 0000000000000000000000000000000000000000..cc631d55fbe40190f4b72fc98bc66070f151d727 --- /dev/null +++ b/ST/evaluate/metrics/xnli/xnli.py @@ -0,0 +1,88 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI benchmark metric. """ + +import datasets + +import evaluate + + +_CITATION = """\ +@InProceedings{conneau2018xnli, + author = "Conneau, Alexis + and Rinott, Ruty + and Lample, Guillaume + and Williams, Adina + and Bowman, Samuel R. + and Schwenk, Holger + and Stoyanov, Veselin", + title = "XNLI: Evaluating Cross-lingual Sentence Representations", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods + in Natural Language Processing", + year = "2018", + publisher = "Association for Computational Linguistics", + location = "Brussels, Belgium", +} +""" + +_DESCRIPTION = """\ +XNLI is a subset of a few thousand examples from MNLI which has been translated +into a 14 different languages (some low-ish resource). As with MNLI, the goal is +to predict textual entailment (does sentence A imply/contradict/neither sentence +B) and is a classification task (given two sentences, predict one of three +labels). +""" + +_KWARGS_DESCRIPTION = """ +Computes XNLI score which is just simple accuracy. +Args: + predictions: Predicted labels. + references: Ground truth labels. +Returns: + 'accuracy': accuracy +Examples: + + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> xnli_metric = evaluate.load("xnli") + >>> results = xnli_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} +""" + + +def simple_accuracy(preds, labels): + return (preds == labels).mean() + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Xnli(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), + "references": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references): + return {"accuracy": simple_accuracy(predictions, references)} diff --git a/ST/evaluate/metrics/xtreme_s/README.md b/ST/evaluate/metrics/xtreme_s/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c93cbfe6fc388666fb6ec5308767fbc083219447 --- /dev/null +++ b/ST/evaluate/metrics/xtreme_s/README.md @@ -0,0 +1,141 @@ +--- +title: XTREME-S +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages. + XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval. +--- + +# Metric Card for XTREME-S + + +## Metric Description + +The XTREME-S metric aims to evaluate model performance on the Cross-lingual TRansfer Evaluation of Multilingual Encoders for Speech (XTREME-S) benchmark. + +This benchmark was designed to evaluate speech representations across languages, tasks, domains and data regimes. It covers 102 languages from 10+ language families, 3 different domains and 4 task families: speech recognition, translation, classification and retrieval. + +## How to Use + +There are two steps: (1) loading the XTREME-S metric relevant to the subset of the benchmark being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant XTREME-S metric** : the subsets of XTREME-S are the following: `mls`, `voxpopuli`, `covost2`, `fleurs-asr`, `fleurs-lang_id`, `minds14` and `babel`. More information about the different subsets can be found on the [XTREME-S benchmark page](https://huggingface.co/datasets/google/xtreme_s). + + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') +``` + +2. **Calculating the metric**: the metric takes two inputs : + +- `predictions`: a list of predictions to score, with each prediction a `str`. + +- `references`: a list of lists of references for each translation, with each reference a `str`. + +```python +>>> references = ["it is sunny here", "paper and pen are essentials"] +>>> predictions = ["it's sunny", "paper pen are essential"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +``` + +It also has two optional arguments: + +- `bleu_kwargs`: a `dict` of keywords to be passed when computing the `bleu` metric for the `covost2` subset. Keywords can be one of `smooth_method`, `smooth_value`, `force`, `lowercase`, `tokenize`, `use_effective_order`. + +- `wer_kwargs`: optional dict of keywords to be passed when computing `wer` and `cer`, which are computed for the `mls`, `fleurs-asr`, `voxpopuli`, and `babel` subsets. Keywords are `concatenate_texts`. + +## Output values + +The output of the metric depends on the XTREME-S subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +- `accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). This is returned for the `fleurs-lang_id` and `minds14` subsets. + +- `f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. It is returned for the `minds14` subset. + +- `wer`: Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. The lower the value, the better the performance of the ASR system, with a WER of 0 being a perfect score (see [WER score](https://huggingface.co/metrics/wer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. + +- `cer`: Character error rate (CER) is similar to WER, but operates on character instead of word. The lower the CER value, the better the performance of the ASR system, with a CER of 0 being a perfect score (see [CER score](https://huggingface.co/metrics/cer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. + +- `bleu`: the BLEU score, calculated according to the SacreBLEU metric approach. It can take any value between 0.0 and 100.0, inclusive, with higher values being better (see [SacreBLEU](https://huggingface.co/metrics/sacrebleu) for more details). This is returned for the `covost2` subset. + + +### Values from popular papers +The [original XTREME-S paper](https://arxiv.org/pdf/2203.10752.pdf) reported average WERs ranging from 9.2 to 14.6, a BLEU score of 20.6, an accuracy of 73.3 and F1 score of 86.9, depending on the subsets of the dataset tested on. + +## Examples + +For the `mls` subset (which outputs `wer` and `cer`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') +>>> references = ["it is sunny here", "paper and pen are essentials"] +>>> predictions = ["it's sunny", "paper pen are essential"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'wer': 0.56, 'cer': 0.27} +``` + +For the `covost2` subset (which outputs `bleu`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'covost2') +>>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] +>>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'bleu': 31.65} +``` + +For the `fleurs-lang_id` subset (which outputs `accuracy`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'fleurs-lang_id') +>>> references = [0, 1, 0, 0, 1] +>>> predictions = [0, 1, 1, 0, 0] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'accuracy': 0.6} + ``` + +For the `minds14` subset (which outputs `f1` and `accuracy`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'minds14') +>>> references = [0, 1, 0, 0, 1] +>>> predictions = [0, 1, 1, 0, 0] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'f1': 0.58, 'accuracy': 0.6} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s). + +While the XTREME-S dataset is meant to represent a variety of languages and tasks, it has inherent biases: it is missing many languages that are important and under-represented in NLP datasets. + +It also has a particular focus on read-speech because common evaluation benchmarks like CoVoST-2 or LibriSpeech evaluate on this type of speech, which results in a mismatch between performance obtained in a read-speech setting and a more noisy setting (in production or live deployment, for instance). + +## Citation + +```bibtex +@article{conneau2022xtreme, + title={XTREME-S: Evaluating Cross-lingual Speech Representations}, + author={Conneau, Alexis and Bapna, Ankur and Zhang, Yu and Ma, Min and von Platen, Patrick and Lozhkov, Anton and Cherry, Colin and Jia, Ye and Rivera, Clara and Kale, Mihir and others}, + journal={arXiv preprint arXiv:2203.10752}, + year={2022} +} +``` + +## Further References + +- [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s) +- [XTREME-S github repository](https://github.com/google-research/xtreme) diff --git a/ST/evaluate/metrics/xtreme_s/app.py b/ST/evaluate/metrics/xtreme_s/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e42af6746ed8c80b9b1aefb132e6afcaf8ef8ef2 --- /dev/null +++ b/ST/evaluate/metrics/xtreme_s/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("xtreme_s", "mls") +launch_gradio_widget(module) diff --git a/ST/evaluate/metrics/xtreme_s/requirements.txt b/ST/evaluate/metrics/xtreme_s/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/ST/evaluate/metrics/xtreme_s/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/ST/evaluate/metrics/xtreme_s/xtreme_s.py b/ST/evaluate/metrics/xtreme_s/xtreme_s.py new file mode 100644 index 0000000000000000000000000000000000000000..b4c052fc50310d1577290f23345e269669d78db1 --- /dev/null +++ b/ST/evaluate/metrics/xtreme_s/xtreme_s.py @@ -0,0 +1,271 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XTREME-S benchmark metric. """ + +from typing import List + +import datasets +from datasets.config import PY_VERSION +from packaging import version +from sklearn.metrics import f1_score + +import evaluate + + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +# TODO(Patrick/Anton) +_CITATION = """\ +""" + +_DESCRIPTION = """\ + XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages. + XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval. +""" + +_KWARGS_DESCRIPTION = """ +Compute XTREME-S evaluation metric associated to each XTREME-S dataset. +Args: + predictions: list of predictions to score. + Each translation should be tokenized into a list of tokens. + references: list of lists of references for each translation. + Each reference should be tokenized into a list of tokens. + bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'. + Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase', + 'tokenize', 'use_effective_order'. + wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'. + Keywords include 'concatenate_texts'. +Returns: depending on the XTREME-S task, one or several of: + "accuracy": Accuracy - for 'fleurs-lang_id', 'minds14' + "f1": F1 score - for 'minds14' + "wer": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' + "cer": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' + "bleu": BLEU score according to the `sacrebleu` metric - for 'covost2' +Examples: + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel' + >>> references = ["it is sunny here", "paper and pen are essentials"] + >>> predictions = ["it's sunny", "paper pen are essential"] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'wer': 0.56, 'cer': 0.27} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'covost2') + >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] + >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'bleu': 31.65} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'fleurs-lang_id') + >>> references = [0, 1, 0, 0, 1] + >>> predictions = [0, 1, 1, 0, 0] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'accuracy': 0.6} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'minds14') + >>> references = [0, 1, 0, 0, 1] + >>> predictions = [0, 1, 1, 0, 0] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'f1': 0.58, 'accuracy': 0.6} +""" + +_CONFIG_NAMES = ["fleurs-asr", "mls", "voxpopuli", "babel", "covost2", "fleurs-lang_id", "minds14"] +SENTENCE_DELIMITER = "" + +try: + from jiwer import transforms as tr + + _jiwer_available = True +except ImportError: + _jiwer_available = False + +if _jiwer_available and version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): + + class SentencesToListOfCharacters(tr.AbstractTransform): + def __init__(self, sentence_delimiter: str = " "): + self.sentence_delimiter = sentence_delimiter + + def process_string(self, s: str): + return list(s) + + def process_list(self, inp: List[str]): + chars = [] + for sent_idx, sentence in enumerate(inp): + chars.extend(self.process_string(sentence)) + if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: + chars.append(self.sentence_delimiter) + return chars + + cer_transform = tr.Compose( + [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] + ) +elif _jiwer_available: + cer_transform = tr.Compose( + [ + tr.RemoveMultipleSpaces(), + tr.Strip(), + tr.ReduceToSingleSentence(SENTENCE_DELIMITER), + tr.ReduceToListOfListOfChars(), + ] + ) +else: + cer_transform = None + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def f1_and_simple_accuracy(preds, labels): + return { + "f1": float(f1_score(y_true=labels, y_pred=preds, average="macro")), + "accuracy": simple_accuracy(preds, labels), + } + + +def bleu( + preds, + labels, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + tokenize=None, + use_effective_order=False, +): + # xtreme-s can only have one label + labels = [[label] for label in labels] + preds = list(preds) + try: + import sacrebleu as scb + except ImportError: + raise ValueError( + "sacrebleu has to be installed in order to apply the bleu metric for covost2." + "You can install it via `pip install sacrebleu`." + ) + + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + + references_per_prediction = len(labels[0]) + if any(len(refs) != references_per_prediction for refs in labels): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in labels] for i in range(references_per_prediction)] + output = scb.corpus_bleu( + preds, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + **(dict(tokenize=tokenize) if tokenize else {}), + ) + return {"bleu": output.score} + + +def wer_and_cer(preds, labels, concatenate_texts, config_name): + try: + from jiwer import compute_measures + except ImportError: + raise ValueError( + f"jiwer has to be installed in order to apply the wer metric for {config_name}." + "You can install it via `pip install jiwer`." + ) + + if concatenate_texts: + wer = compute_measures(labels, preds)["wer"] + + cer = compute_measures(labels, preds, truth_transform=cer_transform, hypothesis_transform=cer_transform)["wer"] + return {"wer": wer, "cer": cer} + else: + + def compute_score(preds, labels, score_type="wer"): + incorrect = 0 + total = 0 + for prediction, reference in zip(preds, labels): + if score_type == "wer": + measures = compute_measures(reference, prediction) + elif score_type == "cer": + measures = compute_measures( + reference, prediction, truth_transform=cer_transform, hypothesis_transform=cer_transform + ) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + return incorrect / total + + return {"wer": compute_score(preds, labels, "wer"), "cer": compute_score(preds, labels, "cer")} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class XtremeS(evaluate.Metric): + def _info(self): + if self.config_name not in _CONFIG_NAMES: + raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") + + pred_type = "int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string" + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + {"predictions": datasets.Value(pred_type), "references": datasets.Value(pred_type)} + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references, bleu_kwargs=None, wer_kwargs=None): + + bleu_kwargs = bleu_kwargs if bleu_kwargs is not None else {} + wer_kwargs = wer_kwargs if wer_kwargs is not None else {} + + if self.config_name == "fleurs-lang_id": + return {"accuracy": simple_accuracy(predictions, references)} + elif self.config_name == "minds14": + return f1_and_simple_accuracy(predictions, references) + elif self.config_name == "covost2": + smooth_method = bleu_kwargs.pop("smooth_method", "exp") + smooth_value = bleu_kwargs.pop("smooth_value", None) + force = bleu_kwargs.pop("force", False) + lowercase = bleu_kwargs.pop("lowercase", False) + tokenize = bleu_kwargs.pop("tokenize", None) + use_effective_order = bleu_kwargs.pop("use_effective_order", False) + return bleu( + preds=predictions, + labels=references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + tokenize=tokenize, + use_effective_order=use_effective_order, + ) + elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]: + concatenate_texts = wer_kwargs.pop("concatenate_texts", False) + return wer_and_cer(predictions, references, concatenate_texts, self.config_name) + else: + raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") diff --git a/ST/evaluate/setup.cfg b/ST/evaluate/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1e2614e9aaa957acf7da9a3586cd7ac6920eae7b --- /dev/null +++ b/ST/evaluate/setup.cfg @@ -0,0 +1,20 @@ +[metadata] +license_file = LICENSE + +[isort] +ensure_newline_before_comments = True +force_grid_wrap = 0 +include_trailing_comma = True +line_length = 119 +lines_after_imports = 2 +multi_line_output = 3 +use_parentheses = True + +[flake8] +ignore = E203, E501, W503 +max-line-length = 119 +exclude = + src/datasets/datasets + src/datasets/metrics +per-file-ignores = + metrics/*:F401 diff --git a/ST/evaluate/setup.py b/ST/evaluate/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..82137e6dcbd8654b376dc44c390f7b332c7b587c --- /dev/null +++ b/ST/evaluate/setup.py @@ -0,0 +1,192 @@ +# Lint as: python3 +""" HuggingFace/Evaluate is an open library for evaluation. + +Note: + + VERSION needs to be formatted following the MAJOR.MINOR.PATCH convention + (we need to follow this convention to be able to retrieve versioned scripts) + +Simple check list for release from AllenNLP repo: https://github.com/allenai/allennlp/blob/master/setup.py + +To create the package for pypi. + +0. Prerequisites: + - Dependencies: + - twine: "pip install twine" + - Create an account in (and join the 'evaluate' project): + - PyPI: https://pypi.org/ + - Test PyPI: https://test.pypi.org/ + +1. Change the version in: + - __init__.py + - setup.py + +2. Commit these changes: "git commit -m 'Release: VERSION'" + +3. Add a tag in git to mark the release: "git tag VERSION -m 'Add tag VERSION for pypi'" + Push the tag to remote: git push --tags origin main + +4. Build both the sources and the wheel. Do not change anything in setup.py between + creating the wheel and the source distribution (obviously). + + First, delete any "build" directory that may exist from previous builds. + + For the wheel, run: "python setup.py bdist_wheel" in the top level directory. + (this will build a wheel for the python version you use to build it). + + For the sources, run: "python setup.py sdist" + You should now have a /dist directory with both .whl and .tar.gz source versions. + +5. Check that everything looks correct by uploading the package to the pypi test server: + + twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/ + + Check that you can install it in a virtualenv/notebook by running: + pip install huggingface_hub fsspec aiohttp + pip install -U tqdm + pip install -i https://testpypi.python.org/pypi evaluate + +6. Upload the final version to actual pypi: + twine upload dist/* -r pypi + +7. Fill release notes in the tag in github once everything is looking hunky-dory. + +8. Change the version in __init__.py and setup.py to X.X.X+1.dev0 (e.g. VERSION=1.18.3 -> 1.18.4.dev0). + Then push the change with a message 'set dev version' +""" + +import os + +from setuptools import find_packages, setup + + +REQUIRED_PKGS = [ + # We need datasets as a backend + "datasets>=2.0.0", + # We use numpy>=1.17 to have np.random.Generator (Dataset shuffling) + "numpy>=1.17", + # For smart caching dataset processing + "dill", + # For performance gains with apache arrow + "pandas", + # for downloading datasets over HTTPS + "requests>=2.19.0", + # progress bars in download and scripts + "tqdm>=4.62.1", + # for fast hashing + "xxhash", + # for better multiprocessing + "multiprocess", + # to get metadata of optional dependencies such as torch or tensorflow for Python versions that don't have it + "importlib_metadata;python_version<'3.8'", + # to save datasets locally or on any filesystem + # minimum 2021.05.0 to have the AbstractArchiveFileSystem + "fsspec[http]>=2021.05.0", + # To get datasets from the Datasets Hub on huggingface.co + "huggingface-hub>=0.7.0", + # Utilities from PyPA to e.g., compare versions + "packaging", + "responses<0.19", +] + +TEMPLATE_REQUIRE = [ + # to populate metric template + "cookiecutter", + # for the gradio widget + "gradio>=3.0.0" +] + +EVALUATOR_REQUIRE = [ + "transformers", + # for bootstrap computations in Evaluator + "scipy>=1.7.1", +] + +TESTS_REQUIRE = [ + # test dependencies + "absl-py", + "charcut>=1.1.1", # for charcut_mt + "cer>=1.2.0", # for characTER + "nltk", # for NIST and probably others + "pytest", + "pytest-datadir", + "pytest-xdist", + # optional dependencies + "tensorflow>=2.3,!=2.6.0,!=2.6.1, <=2.10", + "torch", + # metrics dependencies + "accelerate", # for frugalscore (calls transformers' Trainer) + "bert_score>=0.3.6", + "rouge_score>=0.1.2", + "sacrebleu", + "sacremoses", + "scipy", + "seqeval", + "scikit-learn", + "jiwer", + "sentencepiece", # for bleurt + "transformers", # for evaluator + "mauve-text", + "trectools", + # to speed up pip backtracking + "toml>=0.10.1", + "requests_file>=1.5.1", + "tldextract>=3.1.0", + "texttable>=1.6.3", + "unidecode>=1.3.4", + "Werkzeug>=1.0.1", + "six~=1.15.0", +] + +QUALITY_REQUIRE = ["black~=22.0", "flake8>=3.8.3", "isort>=5.0.0", "pyyaml>=5.3.1"] + + +EXTRAS_REQUIRE = { + "tensorflow": ["tensorflow>=2.2.0,!=2.6.0,!=2.6.1"], + "tensorflow_gpu": ["tensorflow-gpu>=2.2.0,!=2.6.0,!=2.6.1"], + "torch": ["torch"], + "dev": TESTS_REQUIRE + QUALITY_REQUIRE, + "tests": TESTS_REQUIRE, + "quality": QUALITY_REQUIRE, + "docs": [ + # Might need to add doc-builder and some specific deps in the future + "s3fs", + ], + "template": TEMPLATE_REQUIRE, + "evaluator": EVALUATOR_REQUIRE +} + +setup( + name="evaluate", + version="0.4.1.dev0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) + description="HuggingFace community-driven open-source library of evaluation", + long_description=open("README.md", encoding="utf-8").read(), + long_description_content_type="text/markdown", + author="HuggingFace Inc.", + author_email="leandro@huggingface.co", + url="https://github.com/huggingface/evaluate", + download_url="https://github.com/huggingface/evaluate/tags", + license="Apache 2.0", + package_dir={"": "src"}, + packages=find_packages("src"), + entry_points={"console_scripts": ["evaluate-cli=evaluate.commands.evaluate_cli:main"]}, + install_requires=REQUIRED_PKGS, + extras_require=EXTRAS_REQUIRE, + python_requires=">=3.7.0", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], + keywords="metrics machine learning evaluate evaluation", + zip_safe=False, # Required for mypy to find the py.typed file +) diff --git a/ST/evaluate/src/evaluate/__init__.py b/ST/evaluate/src/evaluate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1344b28df26d1f7e6d6bb47bd378c1d281743c9 --- /dev/null +++ b/ST/evaluate/src/evaluate/__init__.py @@ -0,0 +1,51 @@ +# flake8: noqa +# Copyright 2020 The HuggingFace Evaluate Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +# pylint: enable=line-too-long +# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position + +__version__ = "0.4.1.dev0" + +from packaging import version + + +SCRIPTS_VERSION = "main" if version.parse(__version__).is_devrelease else __version__ + +del version + +from .evaluation_suite import EvaluationSuite +from .evaluator import ( + AudioClassificationEvaluator, + AutomaticSpeechRecognitionEvaluator, + Evaluator, + ImageClassificationEvaluator, + QuestionAnsweringEvaluator, + SummarizationEvaluator, + Text2TextGenerationEvaluator, + TextClassificationEvaluator, + TextGenerationEvaluator, + TokenClassificationEvaluator, + TranslationEvaluator, + evaluator, +) +from .hub import push_to_hub +from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo +from .inspect import inspect_evaluation_module, list_evaluation_modules +from .loading import load +from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine +from .saving import save +from .utils import * +from .utils import gradio, logging diff --git a/ST/evaluate/src/evaluate/commands/__init__.py b/ST/evaluate/src/evaluate/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ST/evaluate/src/evaluate/commands/evaluate_cli.py b/ST/evaluate/src/evaluate/commands/evaluate_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..80593c4dfa0f96c8d3ea5ff6131c13c0a94181eb --- /dev/null +++ b/ST/evaluate/src/evaluate/commands/evaluate_cli.py @@ -0,0 +1,137 @@ +import argparse +import os +import subprocess +from pathlib import Path + +from cookiecutter.main import cookiecutter +from huggingface_hub import HfApi, Repository, create_repo + +from evaluate.utils.logging import get_logger + + +logger = get_logger(__name__) + +INSTRUCTIONS = """\ +A new repository for your module "{module_name}" of type "{module_type}" has been created at {output_dir} and pushed to the Hugging Face Hub: {repo_url}. + +Here are the next steps: +- implement the module logic in {module_slug}/{module_slug}.py +- document your module in {module_slug}/README.md +- add test cases for your module in {module_slug}/tests.py +- if your module has any dependencies update them in {module_slug}/requirements.txt + +You can test your module's widget locally by running: + +``` +python {output_dir}/{module_slug}/app.py +``` + +When you are happy with your changes you can push your changes with the following commands to the Hugging Face Hub: + +``` +cd {output_dir}/{module_slug} +git add . +git commit -m "Updating module" +git push +``` + +You should then see the update widget on the Hugging Face Hub: {repo_url} +And you can load your module in Python with the following code: + +``` +from evaluate import load +module = load("{namespace}/{module_slug}") +``` +""" + + +def main(): + parser = argparse.ArgumentParser("HuggingFace Evaluate CLI tool", usage="evaluate-cli []") + subparsers = parser.add_subparsers() + parser_create = subparsers.add_parser("create", help="Create new evaluation module.") + parser_create.add_argument( + "module_name", type=str, help='Pretty name of new evaluation module, e.g. "Recall" or "Exact Match".' + ) + parser_create.add_argument( + "--module_type", + default="metric", + type=str, + help="Type of module, has to be one of [metric|comparison|measurement].", + ) + parser_create.add_argument( + "--dataset_name", default="", type=str, help="Name of dataset if evaluation module is dataset specific." + ) + parser_create.add_argument("--module_description", type=str, help="Short description of evaluation module.") + parser_create.add_argument("--output_dir", default=Path.cwd(), type=str, help="Path to output directory.") + parser_create.add_argument( + "--organization", default=None, type=str, help="Organization on the Hub to push evaluation module to." + ) + parser_create.add_argument("--private", action="store_true", help="Sets evaluation module repository to private.") + args = vars(parser.parse_args()) + + if args["module_type"] not in ["metric", "comparison", "measurement"]: + raise ValueError("The module_type needs to be one of metric, comparison, or measurement") + + if "-" in args["module_name"]: + raise ValueError("Hyphens ('-') are not allowed in module names.") + + output_dir = Path(args["output_dir"]) + organization = args["organization"] + module_slug = args["module_name"].lower().replace(" ", "_") + + if organization is None: + hfapi = HfApi() + namespace = hfapi.whoami()["name"] + else: + namespace = organization + args["namespace"] = namespace + repo_url = f"https://huggingface.co/spaces/{namespace}/{module_slug}" + + try: + create_repo(namespace + "/" + module_slug, repo_type="space", space_sdk="gradio", private=args["private"]) + except Exception as exception: + logger.error( + f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already." + ) + raise exception + subprocess.run( + f"git clone {repo_url}".split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + check=True, + encoding="utf-8", + cwd=output_dir, + env=os.environ.copy(), + ) + + repo = Repository( + local_dir=output_dir / module_slug, + ) + + cookiecutter( + "https://github.com/huggingface/evaluate/", + directory="templates", + no_input=True, + extra_context=args, + output_dir=output_dir, + overwrite_if_exists=True, + ) + + repo.git_add() + repo.git_commit("add module default template") + repo.git_push() + + print( + INSTRUCTIONS.format( + module_name=args["module_name"], + module_type=args["module_type"], + module_slug=module_slug, + namespace=namespace, + repo_url=repo_url, + output_dir=output_dir, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/ST/evaluate/src/evaluate/config.py b/ST/evaluate/src/evaluate/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4909fa251ff82893d7c3c536bb111ae947735a8a --- /dev/null +++ b/ST/evaluate/src/evaluate/config.py @@ -0,0 +1,192 @@ +import importlib +import os +import platform +from pathlib import Path + +from packaging import version + +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +# Metrics +S3_METRICS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics" +CLOUDFRONT_METRICS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/metric" +REPO_METRICS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/metrics/{path}/{name}" +REPO_MEASUREMENTS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/measurements/{path}/{name}" +REPO_COMPARISONS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/comparisons/{path}/{name}" + +# Evaluation module types +EVALUATION_MODULE_TYPES = ["metric", "comparison", "measurement"] + +# Hub +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +HF_LIST_ENDPOINT = HF_ENDPOINT + "/api/spaces?filter={type}" +HUB_EVALUATE_URL = HF_ENDPOINT + "/spaces/{path}/resolve/{revision}/{name}" +HUB_DEFAULT_VERSION = "main" + +PY_VERSION = version.parse(platform.python_version()) + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + +# General environment variables accepted values for booleans +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + + +# Imports +PANDAS_VERSION = version.parse(importlib_metadata.version("pandas")) +PYARROW_VERSION = version.parse(importlib_metadata.version("pyarrow")) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_JAX", "AUTO").upper() + +TORCH_VERSION = "N/A" +TORCH_AVAILABLE = False + +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None + if TORCH_AVAILABLE: + try: + TORCH_VERSION = version.parse(importlib_metadata.version("torch")) + logger.info(f"PyTorch version {TORCH_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling PyTorch because USE_TF is set") + +TF_VERSION = "N/A" +TF_AVAILABLE = False + +if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None + if TF_AVAILABLE: + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for package in [ + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "tensorflow-rocm", + "tensorflow-macos", + ]: + try: + TF_VERSION = version.parse(importlib_metadata.version(package)) + except importlib_metadata.PackageNotFoundError: + continue + else: + break + else: + TF_AVAILABLE = False + if TF_AVAILABLE: + if TF_VERSION.major < 2: + logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.") + TF_AVAILABLE = False + else: + logger.info(f"TensorFlow version {TF_VERSION} available.") +else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +JAX_VERSION = "N/A" +JAX_AVAILABLE = False + +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + JAX_AVAILABLE = importlib.util.find_spec("jax") is not None + if JAX_AVAILABLE: + try: + JAX_VERSION = version.parse(importlib_metadata.version("jax")) + logger.info(f"JAX version {JAX_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling JAX because USE_JAX is set to False") + + +# Cache location +DEFAULT_XDG_CACHE_HOME = "~/.cache" +XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME) +DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface") +HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME)) + +DEFAULT_HF_EVALUATE_CACHE = os.path.join(HF_CACHE_HOME, "evaluate") +HF_EVALUATE_CACHE = Path(os.getenv("HF_EVALUATE_CACHE", DEFAULT_HF_EVALUATE_CACHE)) + +DEFAULT_HF_METRICS_CACHE = os.path.join(HF_CACHE_HOME, "metrics") +HF_METRICS_CACHE = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE)) + +DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") +HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) + +DOWNLOADED_DATASETS_DIR = "downloads" +DEFAULT_DOWNLOADED_EVALUATE_PATH = os.path.join(HF_EVALUATE_CACHE, DOWNLOADED_DATASETS_DIR) +DOWNLOADED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_EVALUATE_PATH", DEFAULT_DOWNLOADED_EVALUATE_PATH)) + +EXTRACTED_EVALUATE_DIR = "extracted" +DEFAULT_EXTRACTED_EVALUATE_PATH = os.path.join(DEFAULT_DOWNLOADED_EVALUATE_PATH, EXTRACTED_EVALUATE_DIR) +EXTRACTED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_EVALUATE_PATH", DEFAULT_EXTRACTED_EVALUATE_PATH)) + +# Download count for the website +HF_UPDATE_DOWNLOAD_COUNTS = ( + os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES +) + +# Offline mode +HF_EVALUATE_OFFLINE = os.environ.get("HF_EVALUATE_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES + + +# File names +LICENSE_FILENAME = "LICENSE" +METRIC_INFO_FILENAME = "metric_info.json" +DATASETDICT_JSON_FILENAME = "dataset_dict.json" + +MODULE_NAME_FOR_DYNAMIC_MODULES = "evaluate_modules" + +HF_HUB_ALLOWED_TASKS = [ + "image-classification", + "translation", + "image-segmentation", + "fill-mask", + "automatic-speech-recognition", + "token-classification", + "sentence-similarity", + "audio-classification", + "question-answering", + "summarization", + "zero-shot-classification", + "table-to-text", + "feature-extraction", + "other", + "multiple-choice", + "text-classification", + "text-to-image", + "text2text-generation", + "zero-shot-image-classification", + "tabular-classification", + "tabular-regression", + "image-to-image", + "tabular-to-text", + "unconditional-image-generation", + "text-retrieval", + "text-to-speech", + "object-detection", + "audio-to-audio", + "text-generation", + "conversational", + "table-question-answering", + "visual-question-answering", + "image-to-text", + "reinforcement-learning", + "voice-activity-detection", + "time-series-forecasting", + "document-question-answering", +] diff --git a/ST/evaluate/src/evaluate/evaluation_suite/__init__.py b/ST/evaluate/src/evaluate/evaluation_suite/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a306d8068a6d66b60fbcc5420bf0cbb334c36305 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluation_suite/__init__.py @@ -0,0 +1,128 @@ +import importlib +import inspect +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, Optional, Union + +from datasets import Dataset, DownloadConfig, DownloadMode, load_dataset +from datasets.utils.version import Version + +from ..evaluator import evaluator +from ..loading import evaluation_module_factory +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +@dataclass +class SubTask: + task_type: str + data: Optional[Union[str, Dataset]] = None + subset: Optional[str] = None + split: Optional[str] = None + data_preprocessor: Optional[Callable] = None + args_for_task: Optional[dict] = None + + def __post_init__(self): + if type(self.task_type) is not str: + raise ValueError(f"'task_type' must be type 'str', got {type(self.task_type)}") + if type(self.data) not in [Dataset, str]: + raise ValueError( + f"'data' must be an already-instantiated Dataset object or type 'str', got {type(self.data)}" + ) + if self.subset and type(self.subset) is not str: + raise ValueError(f"'subset' must be type 'str', got {type(self.subset)}") + if self.split and type(self.split) is not str: + raise ValueError(f"'split' must be type 'str', got {type(self.split)}") + if self.data_preprocessor and not callable(self.data_preprocessor): + raise ValueError(f"'data_preprocessor' must be a Callable', got {self.data_preprocessor}") + if self.args_for_task and type(self.args_for_task) is not dict: + raise ValueError(f"'args_for_task' must be type 'dict', got {type(self.args_for_task)}") + + +def import_main_class(module_path): + """Import a module at module_path and return the EvaluationSuite class""" + module = importlib.import_module(module_path) + + module_main_cls = None + for name, obj in module.__dict__.items(): + if isinstance(obj, type) and obj.__name__ == "Suite": + if inspect.isabstract(obj): + continue + module_main_cls = obj + break + + return module_main_cls + + +class EvaluationSuite: + """ + This class instantiates an evaluation suite made up of multiple tasks, where each task consists of a dataset and + an associated metric, and runs evaluation on a model or pipeline. Evaluation suites can be a Python script found + either locally or uploaded as a Space on the Hugging Face Hub. + Usage: + ```python + from evaluate import EvaluationSuite + suite = EvaluationSuite.load("evaluate/evaluation-suite-ci") + results = suite.run("lvwerra/distilbert-imdb") + ``` + """ + + def __init__(self, name): + self.name = name + + @staticmethod + def load( + path: str, + download_mode: Optional[DownloadMode] = None, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + ): + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + evaluation_module = evaluation_module_factory( + path, module_type=None, revision=revision, download_config=download_config, download_mode=download_mode + ) + name = Path(path).stem + evaluation_cls = import_main_class(evaluation_module.module_path) + evaluation_instance = evaluation_cls(name) + + return evaluation_instance + + def __repr__(self): + self.tasks = [str(task) for task in self.suite] + return f'EvaluationSuite name: "{self.name}", ' f"Tasks: {self.tasks})" + + def assert_suite_nonempty(self): + if not self.suite: + raise ValueError( + "No evaluation tasks found. The EvaluationSuite must include at least one SubTask definition." + ) + + def run( + self, model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"] # noqa: F821 + ) -> Dict[str, float]: + + self.assert_suite_nonempty() + + results_all = [] + for task in self.suite: + + task_name = task.data + + if task.data_preprocessor: # task requires extra preprocessing + ds = load_dataset(task.data, name=task.subset, split=task.split) + task.data = ds.map(task.data_preprocessor) + + task_evaluator = evaluator(task.task_type) + args_for_task = task.args_for_task + args_for_task["model_or_pipeline"] = model_or_pipeline + args_for_task["data"] = task.data + args_for_task["subset"] = task.subset + args_for_task["split"] = task.split + results = task_evaluator.compute(**args_for_task) + + results["task_name"] = task_name + "/" + task.subset if task.subset else task_name + results["data_preprocessor"] = str(task.data_preprocessor) if task.data_preprocessor is not None else None + results_all.append(results) + return results_all diff --git a/ST/evaluate/src/evaluate/evaluator/__init__.py b/ST/evaluate/src/evaluate/evaluator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a2fe4be8a1332417fb8515f019c1b7e8c41a58bf --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/__init__.py @@ -0,0 +1,140 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + from transformers.pipelines import SUPPORTED_TASKS as SUPPORTED_PIPELINE_TASKS + from transformers.pipelines import TASK_ALIASES + from transformers.pipelines import check_task as check_pipeline_task + + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from typing import Dict, List + +from .audio_classification import AudioClassificationEvaluator +from .automatic_speech_recognition import AutomaticSpeechRecognitionEvaluator +from .base import Evaluator +from .image_classification import ImageClassificationEvaluator +from .question_answering import QuestionAnsweringEvaluator +from .text2text_generation import SummarizationEvaluator, Text2TextGenerationEvaluator, TranslationEvaluator +from .text_classification import TextClassificationEvaluator +from .text_generation import TextGenerationEvaluator +from .token_classification import TokenClassificationEvaluator + + +SUPPORTED_EVALUATOR_TASKS = { + "text-classification": { + "implementation": TextClassificationEvaluator, + "default_metric_name": "accuracy", + }, + "image-classification": { + "implementation": ImageClassificationEvaluator, + "default_metric_name": "accuracy", + }, + "question-answering": { + "implementation": QuestionAnsweringEvaluator, + "default_metric_name": "squad", + }, + "token-classification": { + "implementation": TokenClassificationEvaluator, + "default_metric_name": "seqeval", + }, + "text-generation": { + "implementation": TextGenerationEvaluator, + "default_metric_name": "word_count", + }, + "text2text-generation": { + "implementation": Text2TextGenerationEvaluator, + "default_metric_name": "bleu", + }, + "summarization": { + "implementation": SummarizationEvaluator, + "default_metric_name": "rouge", + }, + "translation": { + "implementation": TranslationEvaluator, + "default_metric_name": "bleu", + }, + "automatic-speech-recognition": { + "implementation": AutomaticSpeechRecognitionEvaluator, + "default_metric_name": "wer", + }, + "audio-classification": { + "implementation": AudioClassificationEvaluator, + "default_metric_name": "accuracy", + }, +} + + +def get_supported_tasks() -> List[str]: + """ + Returns a list of supported task strings. + """ + return list(SUPPORTED_EVALUATOR_TASKS.keys()) + + +def check_task(task: str) -> Dict: + """ + Checks an incoming task string, to validate it's correct and returns the default Evaluator class and default metric + name. It first performs a check to validata that the string is a valid `Pipeline` task, then it checks if it's a + valid `Evaluator` task. `Evaluator` tasks are a substet of `Pipeline` tasks. + Args: + task (`str`): + The task defining which evaluator will be returned. Currently accepted tasks are: + - `"image-classification"` + - `"question-answering"` + - `"text-classification"` (alias `"sentiment-analysis"` available) + - `"token-classification"` + Returns: + task_defaults: `dict`, contains the implementasion class of a give Evaluator and the default metric name. + """ + if task in TASK_ALIASES: + task = TASK_ALIASES[task] + if not check_pipeline_task(task): + raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") + if task in SUPPORTED_EVALUATOR_TASKS.keys() and task in SUPPORTED_PIPELINE_TASKS.keys(): + return SUPPORTED_EVALUATOR_TASKS[task] + raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") + + +def evaluator(task: str = None) -> Evaluator: + """ + Utility factory method to build an [`Evaluator`]. + Evaluators encapsulate a task and a default metric name. They leverage `pipeline` functionality from `transformers` + to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task. + Args: + task (`str`): + The task defining which evaluator will be returned. Currently accepted tasks are: + - `"image-classification"`: will return a [`ImageClassificationEvaluator`]. + - `"question-answering"`: will return a [`QuestionAnsweringEvaluator`]. + - `"text-classification"` (alias `"sentiment-analysis"` available): will return a [`TextClassificationEvaluator`]. + - `"token-classification"`: will return a [`TokenClassificationEvaluator`]. + Returns: + [`Evaluator`]: An evaluator suitable for the task. + Examples: + ```python + >>> from evaluate import evaluator + >>> # Sentiment analysis evaluator + >>> evaluator("sentiment-analysis") + ```""" + if not TRANSFORMERS_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `transformers`. Run `pip install evaluate[transformers]`." + ) + targeted_task = check_task(task) + evaluator_class = targeted_task["implementation"] + default_metric_name = targeted_task["default_metric_name"] + return evaluator_class(task=task, default_metric_name=default_metric_name) diff --git a/ST/evaluate/src/evaluate/evaluator/audio_classification.py b/ST/evaluate/src/evaluate/evaluator/audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..685fb9fd8515f8506b89e9375948fea181f79a8f --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/audio_classification.py @@ -0,0 +1,151 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + + + + Remember that, in order to process audio files, you need ffmpeg installed (https://ffmpeg.org/download.html) + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> task_evaluator = evaluator("audio-classification") + >>> data = load_dataset("superb", 'ks', split="test[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"", + >>> data=data, + >>> label_column="label", + >>> input_column="file", + >>> metric="accuracy", + >>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"} + >>> ) + ``` + + + + The evaluator supports raw audio data as well, in the form of a numpy array. However, be aware that calling + the audio column automatically decodes and resamples the audio files, which can be slow for large datasets. + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> task_evaluator = evaluator("audio-classification") + >>> data = load_dataset("superb", 'ks', split="test[:40]") + >>> data = data.map(lambda example: {"audio": example["audio"]["array"]}) + >>> results = task_evaluator.compute( + >>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"", + >>> data=data, + >>> label_column="label", + >>> input_column="audio", + >>> metric="accuracy", + >>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"} + >>> ) + ``` +""" + + +class AudioClassificationEvaluator(Evaluator): + """ + Audio classification evaluator. + This audio classification evaluator can currently be loaded from [`evaluator`] using the default task name + `audio-classification`. + Methods in this class assume a data format compatible with the [`transformers.AudioClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="audio-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + pred_label = [max(pred, key=lambda x: x["score"])["label"] for pred in predictions] + pred_label = [label_mapping[pred] if label_mapping is not None else pred for pred in pred_label] + + return {"predictions": pred_label} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "file", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + + """ + input_column (`str`, defaults to `"file"`): + The name of the column containing either the audio files or a raw waveform, represented as a numpy array, in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + label_mapping=label_mapping, + ) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py b/ST/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..ee423826cdd7bac384080b3db8a369cc59a53283 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py @@ -0,0 +1,112 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("automatic-speech-recognition") + >>> data = load_dataset("mozilla-foundation/common_voice_11_0", "en", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="https://huggingface.co/openai/whisper-tiny.en", + >>> data=data, + >>> input_column="path", + >>> label_column="sentence", + >>> metric="wer", + >>> ) + ``` +""" + + +class AutomaticSpeechRecognitionEvaluator(Evaluator): + """ + Automatic speech recognition evaluator. + This automatic speech recognition evaluator can currently be loaded from [`evaluator`] using the default task name + `automatic-speech-recognition`. + Methods in this class assume a data format compatible with the [`AutomaticSpeechRecognitionPipeline`]. + """ + + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="automatic-speech-recognition", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + return {"predictions": [pred["text"] for pred in predictions]} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "path", + label_column: str = "sentence", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, defaults to `"path"`): + the name of the column containing the input audio path in the dataset specified by `data`. + label_column (`str`, defaults to `"sentence"`): + the name of the column containing the labels in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. + """ + + if generation_kwargs is not None: + self.PIPELINE_KWARGS.update(generation_kwargs) + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + ) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/base.py b/ST/evaluate/src/evaluate/evaluator/base.py new file mode 100644 index 0000000000000000000000000000000000000000..04a370eb46efe38c82216213f09358f3c3e4eab2 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/base.py @@ -0,0 +1,544 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from numbers import Number +from typing import Any, Callable, Dict, List, Optional, Union + +# Lint as: python3 +from datasets import Dataset, load_dataset + +from evaluate.evaluator.utils import choose_split + + +try: + from scipy.stats import bootstrap + + SCIPY_AVAILABLE = True +except ImportError: + SCIPY_AVAILABLE = False + +try: + import transformers + from transformers import Pipeline, pipeline + + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from time import perf_counter + +from typing_extensions import Literal + +from ..loading import load +from ..module import EvaluationModule +from ..utils.logging import get_logger +from .utils import DatasetColumn + + +logger = get_logger(__name__) + + +EVALUTOR_COMPUTE_START_DOCSTRING = r""" + Compute the metric for a given pipeline and dataset combination. + Args: + model_or_pipeline (`str` or `Pipeline` or `Callable` or `PreTrainedModel` or `TFPreTrainedModel`, defaults to `None`): + If the argument in not specified, we initialize the default pipeline for the task (in this case + `text-classification` or its alias - `sentiment-analysis`). If the argument is of the type `str` or + is a model instance, we use it to initialize a new `Pipeline` with the given model. Otherwise we assume the + argument specifies a pre-initialized pipeline. + data (`str` or `Dataset`, defaults to `None`): + Specifies the dataset we will run evaluation on. If it is of type `str`, we treat it as the dataset + name, and load it. Otherwise we assume it represents a pre-loaded dataset. + subset (`str`, defaults to `None`): + Defines which dataset subset to load. If `None` is passed the default subset is loaded. + split (`str`, defaults to `None`): + Defines which dataset split to load. If `None` is passed, infers based on the `choose_split` function. + metric (`str` or `EvaluationModule`, defaults to `None`): + Specifies the metric we use in evaluator. If it is of type `str`, we treat it as the metric name, and + load it. Otherwise we assume it represents a pre-loaded metric. + tokenizer (`str` or `PreTrainedTokenizer`, *optional*, defaults to `None`): + Argument can be used to overwrite a default tokenizer if `model_or_pipeline` represents a model for + which we build a pipeline. If `model_or_pipeline` is `None` or a pre-initialized pipeline, we ignore + this argument. + strategy (`Literal["simple", "bootstrap"]`, defaults to "simple"): + specifies the evaluation strategy. Possible values are: + - `"simple"` - we evaluate the metric and return the scores. + - `"bootstrap"` - on top of computing the metric scores, we calculate the confidence interval for each + of the returned metric keys, using `scipy`'s `bootstrap` method + https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html. + confidence_level (`float`, defaults to `0.95`): + The `confidence_level` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. + n_resamples (`int`, defaults to `9999`): + The `n_resamples` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. + device (`int`, defaults to `None`): + Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive + integer will run the model on the associated CUDA device ID. If `None` is provided it will be inferred and + CUDA:0 used if available, CPU otherwise. + random_state (`int`, *optional*, defaults to `None`): + The `random_state` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. Useful for + debugging. +""" + +EVALUATOR_COMPUTE_RETURN_DOCSTRING = r""" + Return: + A `Dict`. The keys represent metric keys calculated for the `metric` spefied in function arguments. For the + `"simple"` strategy, the value is the metric score. For the `"bootstrap"` strategy, the value is a `Dict` + containing the score, the confidence interval and the standard error calculated for each metric key. +""" + + +class Evaluator(ABC): + """ + The [`Evaluator`] class is the class from which all evaluators inherit. Refer to this class for methods shared across + different evaluators. + Base class implementing evaluator operations. + """ + + PIPELINE_KWARGS = {} + METRIC_KWARGS = {} + + def __init__(self, task: str, default_metric_name: str = None): + if not TRANSFORMERS_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `transformers`. Run `pip install evaluate[evaluator]`." + ) + if not SCIPY_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `scipy>=1.7.1`. Run `pip install evaluate[evaluator]`." + ) + self.task = task + self.default_metric_name = default_metric_name + + @staticmethod + def _compute_confidence_interval( + metric, + metric_inputs, + metric_keys: List[str], + confidence_level: float = 0.95, + n_resamples: int = 9999, + random_state: Optional[int] = None, + ) -> Dict[str, Any]: + """ + A utility function enabling the confidence interval calculation for metrics computed + by the evaluator based on `scipy`'s `bootstrap` method. + """ + + # bootstrap only works with functions that use args and no kwargs + def build_args_metric(metric, key, **kwargs): + def args_metric(*args): + return metric.compute(**{k: v for k, v in zip(kwargs.keys(), args)})[key] + + return args_metric + + bootstrap_dict = {} + for key in metric_keys: + bs = bootstrap( + data=list(metric_inputs.values()), + statistic=build_args_metric(metric, key, **metric_inputs), + paired=True, + vectorized=False, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + bootstrap_dict[key] = { + "confidence_interval": (bs.confidence_interval.low, bs.confidence_interval.high), + "standard_error": bs.standard_error, + } + return bootstrap_dict + + @staticmethod + def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> Dict[str, Any]: + """ + A utility function computing time performance metrics: + - `total_time_in_seconds` - pipeline inference runtime for the evaluation data in seconds, + - `samples_per_second` - pipeline throughput in the number of samples per second. + - `latency_in_seconds` - pipeline inference runtime for the evaluation data in seconds per sample, + + """ + latency = end_time - start_time + throughput = num_samples / latency + latency_sample = 1.0 / throughput + + return { + "total_time_in_seconds": latency, + "samples_per_second": throughput, + "latency_in_seconds": latency_sample, + } + + @staticmethod + def _infer_device() -> int: + """Helper function to check if GPU or CPU is available for inference.""" + # try infer with torch first + try: + import torch + + if torch.cuda.is_available(): + device = 0 # first GPU + else: + device = -1 # CPU + except ImportError: + # if not available try TF + try: + import tensorflow as tf + + if len(tf.config.list_physical_devices("GPU")) > 0: + device = 0 # first GPU + else: + device = -1 # CPU + except ImportError: + device = -1 + + if device == -1: + logger.info("No GPU found. The default device for pipeline inference is set to CPU.") + else: + logger.info("GPU found. The default device for pipeline inference is set to GPU (CUDA:0).") + + return device + + @abstractmethod + def predictions_processor(self, *args, **kwargs): + """ + A core method of the `Evaluator` class, which processes the pipeline outputs for compatibility with the metric. + """ + raise NotImplementedError() + + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Dict[str, float]: + + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data(data=data, input_column=input_column, label_column=label_column) + pipe = self.prepare_pipeline( + model_or_pipeline=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, label_mapping) + + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + # TODO: To clarify why `wer` and `cer` return float + # even though metric.compute contract says that it + # returns Optional[dict]. + if type(metric_results) == float: + metric_results = {metric.name: metric_results} + + result.update(metric_results) + result.update(perf_results) + + return result + + @staticmethod + def check_for_mismatch_in_device_setup(device, model_or_pipeline): + if device is not None and device != -1 and isinstance(model_or_pipeline, Pipeline): + if model_or_pipeline.device.type == "cpu": + raise ValueError( + "The value of the `device` kwarg passed to `compute` suggests that this pipe should be run on an " + "accelerator, but the pipe was instantiated on CPU. Pass `device` to the pipeline during " + "initialization to use an accelerator, or pass `device=None` to `compute`. " + ) + elif device != model_or_pipeline.device.index: + raise ValueError( + f"This pipeline was instantiated on device {model_or_pipeline.device.index} but device={device} was passed to `compute`." + ) + + def check_required_columns(self, data: Union[str, Dataset], columns_names: Dict[str, str]): + """ + Ensure the columns required for the evaluation are present in the dataset. + + Args: + data (`str` or [`Dataset`]): + Specifies the dataset we will run evaluation on. + columns_names (`List[str]`): + List of column names to check in the dataset. The keys are the arguments to the [`evaluate.EvaluationModule.compute`] method, + while the values are the column names to check. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from evaluate import evaluator + >>> data = load_dataset("rotten_tomatoes', split="train") + >>> evaluator.check_required_columns(data, {"input_column": "text", "label_column": "label"}) + ``` + """ + for input_name, column_name in columns_names.items(): + if column_name not in data.column_names: + raise ValueError( + f"Invalid `{input_name}` {column_name} specified. The dataset contains the following columns: {data.column_names}." + ) + + @staticmethod + def get_dataset_split(data, subset=None, split=None): + """ + Infers which split to use if `None` is given. + + Args: + data (`str`): + Name of dataset. + subset (`str`): + Name of config for datasets with multiple configurations (e.g. 'glue/cola'). + split (`str`, defaults to `None`): + Split to use. + Returns: + `split`: `str` containing which split to use + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").get_dataset_split(data="rotten_tomatoes") + WARNING:evaluate.evaluator.base:Dataset split not defined! Automatically evaluating with split: TEST + 'test' + ``` + """ + if split is None: + split = choose_split(data, subset) + logger.warning(f"Dataset split not defined! Automatically evaluating with split: {split.upper()}") + return split + + def load_data(self, data: Union[str, Dataset], subset: str = None, split: str = None): + """ + Load dataset with given subset and split. + Args: + data ([`Dataset`] or `str`, defaults to `None`): + Specifies the dataset we will run evaluation on. If it is of + type `str`, we treat it as the dataset name, and load it. Otherwise we assume it represents a pre-loaded dataset. + subset (`str`, defaults to `None`): + Specifies dataset subset to be passed to `name` in `load_dataset`. To be + used with datasets with several configurations (e.g. glue/sst2). + split (`str`, defaults to `None`): + User-defined dataset split by name (e.g. train, validation, test). Supports slice-split (`test[:n]`). + If not defined and data is a `str` type, will automatically select the best one via `choose_split()`. + Returns: + data ([`Dataset`]): Loaded dataset which will be used for evaluation. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").load_data(data="rotten_tomatoes", split="train") + Dataset({ + features: ['text', 'label'], + num_rows: 8530 + }) + ``` + """ + if isinstance(data, str): + split = self.get_dataset_split(data, subset, split) + data = load_dataset(data, name=subset, split=split) + return data + elif isinstance(data, Dataset): + if split is not None or subset is not None: + logger.warning("`data` is a preloaded Dataset! Ignoring `subset` and `split`.") + return data + else: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + + def prepare_data(self, data: Dataset, input_column: str, label_column: str, *args, **kwargs): + """ + Prepare data. + + Args: + data ([`Dataset`]): + Specifies the dataset we will run evaluation on. + input_column (`str`, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + second_input_column(`str`, *optional*): + The name of the column containing the second text feature if there is one. Otherwise, set to `None`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + Returns: + `dict`: metric inputs. + `list`: pipeline inputs. + + Example: + + ```py + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> ds = load_dataset("rotten_tomatoes", split="train") + >>> evaluator("text-classification").prepare_data(ds, input_column="text", second_input_column=None, label_column="label") + ``` + """ + + self.check_required_columns(data, {"input_column": input_column, "label_column": label_column}) + + return {"references": data[label_column]}, DatasetColumn(data, input_column) + + def prepare_pipeline( + self, + model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"], # noqa: F821 + tokenizer: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + feature_extractor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + device: int = None, + ): + """ + Prepare pipeline. + + Args: + model_or_pipeline (`str` or [`~transformers.Pipeline`] or `Callable` or [`~transformers.PreTrainedModel`] or [`~transformers.TFPreTrainedModel`], defaults to `None`): + If the argument in not specified, we initialize the default pipeline for the task. If the argument is of the type `str` or + is a model instance, we use it to initialize a new [`~transformers.Pipeline`] with the given model. Otherwise we assume the + argument specifies a pre-initialized pipeline. + preprocessor ([`~transformers.PreTrainedTokenizerBase`] or [`~transformers.FeatureExtractionMixin`], *optional*, defaults to `None`): + Argument can be used to overwrite a default preprocessor if `model_or_pipeline` represents a model for + which we build a pipeline. If `model_or_pipeline` is `None` or a pre-initialized pipeline, we ignore + this argument. + Returns: + The initialized pipeline. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").prepare_pipeline(model_or_pipeline="distilbert-base-uncased") + ``` + """ + + if device is None: + device = self._infer_device() + + if ( + isinstance(model_or_pipeline, str) + or isinstance(model_or_pipeline, transformers.PreTrainedModel) + or isinstance(model_or_pipeline, transformers.TFPreTrainedModel) + ): + pipe = pipeline( + self.task, + model=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + else: + if model_or_pipeline is None: + pipe = pipeline(self.task, device=device) + else: + pipe = model_or_pipeline + if tokenizer is not None and feature_extractor is not None: + logger.warning("Ignoring the value of the preprocessor argument (`tokenizer` or `feature_extractor`).") + if (pipe.task != self.task) and not (self.task == "translation" and pipe.task.startswith("translation")): + raise ValueError( + f"Incompatible `model_or_pipeline`. Please specify `model_or_pipeline` compatible with the `{self.task}` task." + ) + return pipe + + def prepare_metric(self, metric: Union[str, EvaluationModule]): + """ + Prepare metric. + + Args: + metric (`str` or [`EvaluationModule`], defaults to `None`): + Specifies the metric we use in evaluator. If it is of type `str`, we treat it as the metric name, and + load it. Otherwise we assume it represents a pre-loaded metric. + + Returns: + The loaded metric. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").prepare_metric("accuracy") + ``` + """ + # Prepare metric. + if metric is None: + if self.default_metric_name is None: + raise ValueError( + "`Evaluator` doesn't specify a default metric. Please specify a valid `metric` argument." + ) + metric = load(self.default_metric_name) + elif isinstance(metric, str): + metric = load(metric) + + return metric + + def call_pipeline(self, pipe, *args, **kwargs): + start_time = perf_counter() + pipe_output = pipe(*args, **kwargs, **self.PIPELINE_KWARGS) + end_time = perf_counter() + return pipe_output, self._compute_time_perf(start_time, end_time, len(pipe_output)) + + def compute_metric( + self, + metric: EvaluationModule, + metric_inputs: Dict, + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + random_state: Optional[int] = None, + ): + """Compute and return metrics.""" + result = metric.compute(**metric_inputs, **self.METRIC_KWARGS) + + if strategy == "bootstrap": + metric_keys = result.keys() + bootstrap_dict = self._compute_confidence_interval( + metric, + metric_inputs, + metric_keys, + confidence_level, + n_resamples, + random_state, + ) + for key in metric_keys: + bootstrap_dict[key]["score"] = result[key] + + return bootstrap_dict + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/image_classification.py b/ST/evaluate/src/evaluate/evaluator/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..82831458bb8789ce9c9418d6c19d4af4ba5b35a2 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/image_classification.py @@ -0,0 +1,119 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("image-classification") + >>> data = load_dataset("beans", split="test[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="nateraw/vit-base-beans", + >>> data=data, + >>> label_column="labels", + >>> metric="accuracy", + >>> label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2}, + >>> strategy="bootstrap" + >>> ) + ``` +""" + + +class ImageClassificationEvaluator(Evaluator): + """ + Image classification evaluator. + This image classification evaluator can currently be loaded from [`evaluator`] using the default task name + `image-classification`. + Methods in this class assume a data format compatible with the [`ImageClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="image-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + pred_label = [max(pred, key=lambda x: x["score"])["label"] for pred in predictions] + pred_label = [label_mapping[pred] if label_mapping is not None else pred for pred in pred_label] + + return {"predictions": pred_label} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "image", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + + """ + input_column (`str`, defaults to `"image"`): + The name of the column containing the images as PIL ImageFile in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + label_mapping=label_mapping, + ) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/question_answering.py b/ST/evaluate/src/evaluate/evaluator/question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..99b4190eebdda4e90617d0979fe23af2965d3204 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/question_answering.py @@ -0,0 +1,239 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union + +# Lint as: python3 +from datasets import Dataset + + +try: + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from ..utils.logging import get_logger +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumn + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +logger = get_logger(__name__) + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("question-answering") + >>> data = load_dataset("squad", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad", + >>> data=data, + >>> metric="squad", + >>> ) + ``` + + + + Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass `squad_v2_format=True` to + the compute() call. + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("question-answering") + >>> data = load_dataset("squad_v2", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2", + >>> data=data, + >>> metric="squad_v2", + >>> squad_v2_format=True, + >>> ) + ``` +""" + + +class QuestionAnsweringEvaluator(Evaluator): + """ + Question answering evaluator. This evaluator handles + [**extractive** question answering](https://huggingface.co/docs/transformers/task_summary#extractive-question-answering), + where the answer to the question is extracted from a context. + + This question answering evaluator can currently be loaded from [`evaluator`] using the default task name + `question-answering`. + + Methods in this class assume a data format compatible with the + [`~transformers.QuestionAnsweringPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="question-answering", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def prepare_data( + self, data: Dataset, question_column: str, context_column: str, id_column: str, label_column: str + ): + """Prepare data.""" + if data is None: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + self.check_required_columns( + data, + { + "question_column": question_column, + "context_column": context_column, + "id_column": id_column, + "label_column": label_column, + }, + ) + + metric_inputs = dict() + metric_inputs["references"] = [ + {"id": element[id_column], "answers": element[label_column]} for element in data + ] + + return metric_inputs, { + "question": DatasetColumn(data, question_column), + "context": DatasetColumn(data, context_column), + } + + def is_squad_v2_format(self, data: Dataset, label_column: str = "answers"): + """ + Check if the provided dataset follows the squad v2 data schema, namely possible samples where the answer is not in the context. + In this case, the answer text list should be `[]`. + """ + original_num_rows = data.num_rows + nonempty_num_rows = data.filter( + lambda x: len(x[label_column]["text"]) > 0, load_from_cache_file=False + ).num_rows + if original_num_rows > nonempty_num_rows: + return True + else: + return False + + def predictions_processor(self, predictions: List, squad_v2_format: bool, ids: List): + result = [] + for i in range(len(predictions)): + pred = {"prediction_text": predictions[i]["answer"], "id": ids[i]} + if squad_v2_format: + pred["no_answer_probability"] = predictions[i]["score"] + result.append(pred) + return {"predictions": result} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + question_column: str = "question", + context_column: str = "context", + id_column: str = "id", + label_column: str = "answers", + squad_v2_format: Optional[bool] = None, + ) -> Tuple[Dict[str, float], Any]: + """ + question_column (`str`, defaults to `"question"`): + The name of the column containing the question in the dataset specified by `data`. + context_column (`str`, defaults to `"context"`): + The name of the column containing the context in the dataset specified by `data`. + id_column (`str`, defaults to `"id"`): + The name of the column containing the identification field of the question and answer pair in the + dataset specified by `data`. + label_column (`str`, defaults to `"answers"`): + The name of the column containing the answers in the dataset specified by `data`. + squad_v2_format (`bool`, *optional*, defaults to `None`): + Whether the dataset follows the format of squad_v2 dataset. This is the case when the provided dataset + has questions where the answer is not in the context, more specifically when are answers as + `{"text": [], "answer_start": []}` in the answer column. If all questions have at least one answer, this parameter + should be set to `False`. If this parameter is not provided, the format will be automatically inferred. + """ + result = {} + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, + question_column=question_column, + context_column=context_column, + id_column=id_column, + label_column=label_column, + ) + + if squad_v2_format is None: + squad_v2_format = self.is_squad_v2_format(data=data, label_column=label_column) + logger.warning( + f"`squad_v2_format` parameter not provided to QuestionAnsweringEvaluator.compute(). Automatically inferred `squad_v2_format` as {squad_v2_format}." + ) + pipe = self.prepare_pipeline(model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device) + + metric = self.prepare_metric(metric) + + if squad_v2_format and metric.name == "squad": + logger.warning( + "The dataset has SQuAD v2 format but you are using the SQuAD metric. Consider passing the 'squad_v2' metric." + ) + if not squad_v2_format and metric.name == "squad_v2": + logger.warning( + "The dataset has SQuAD v1 format but you are using the SQuAD v2 metric. Consider passing the 'squad' metric." + ) + + if squad_v2_format: + self.PIPELINE_KWARGS["handle_impossible_answer"] = True + else: + self.PIPELINE_KWARGS["handle_impossible_answer"] = False + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, **pipe_inputs) + predictions = self.predictions_processor(predictions, squad_v2_format=squad_v2_format, ids=data[id_column]) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/text2text_generation.py b/ST/evaluate/src/evaluate/evaluator/text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..6dfd2c035695b38c1e4f0d9d4929b12c6be30920 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/text2text_generation.py @@ -0,0 +1,267 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION_KWARGS = r""" + input_column (`str`, defaults to `"text"`): + the name of the column containing the input text in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + the name of the column containing the labels in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. +""" + +TEXT2TEXT_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("text2text-generation") + >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="facebook/bart-large-cnn", + >>> data=data, + >>> input_column="article", + >>> label_column="highlights", + >>> metric="rouge", + >>> ) + ``` +""" + +SUMMARIZATION_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("summarization") + >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="facebook/bart-large-cnn", + >>> data=data, + >>> input_column="article", + >>> label_column="highlights", + >>> ) + ``` +""" + + +TRANSLATION_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("translation") + >>> data = load_dataset("wmt19", "fr-de", split="validation[:40]") + >>> data = data.map(lambda x: {"text": x["translation"]["de"], "label": x["translation"]["fr"]}) + >>> results = task_evaluator.compute( + >>> model_or_pipeline="Helsinki-NLP/opus-mt-de-fr", + >>> data=data, + >>> ) + ``` +""" + + +class Text2TextGenerationEvaluator(Evaluator): + """ + Text2Text generation evaluator. + This Text2Text generation evaluator can currently be loaded from [`evaluator`] using the default task name + `text2text-generation`. + Methods in this class assume a data format compatible with the [`~transformers.Text2TextGenerationPipeline`]. + """ + + PREDICTION_PREFIX = "generated" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="text2text-generation", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + return {"predictions": [pred[f"{self.PREDICTION_PREFIX}_text"] for pred in predictions]} + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + TEXT2TEXT_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + if generation_kwargs is not None: + self.PIPELINE_KWARGS.update(generation_kwargs) + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + ) + + return result + + +class SummarizationEvaluator(Text2TextGenerationEvaluator): + """ + Text summarization evaluator. + This text summarization evaluator can currently be loaded from [`evaluator`] using the default task name + `summarization`. + Methods in this class assume a data format compatible with the [`SummarizationEvaluator`]. + """ + + PREDICTION_PREFIX = "summary" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="summarization", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + SUMMARIZATION_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + generation_kwargs=generation_kwargs, + ) + + return result + + +class TranslationEvaluator(Text2TextGenerationEvaluator): + """ + Translation evaluator. + This translation generation evaluator can currently be loaded from [`evaluator`] using the default task name + `translation`. + Methods in this class assume a data format compatible with the [`~transformers.TranslationPipeline`]. + """ + + PREDICTION_PREFIX = "translation" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="translation", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + TRANSLATION_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + generation_kwargs=generation_kwargs, + ) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/text_classification.py b/ST/evaluate/src/evaluate/evaluator/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..200eb01d70336148db473edebebc96e3137c5799 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/text_classification.py @@ -0,0 +1,160 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset, load_dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumnPair + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("text-classification") + >>> data = load_dataset("imdb", split="test[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli", + >>> data=data, + >>> metric="accuracy", + >>> label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0}, + >>> strategy="bootstrap", + >>> n_resamples=10, + >>> random_state=0 + >>> ) + ``` +""" + + +class TextClassificationEvaluator(Evaluator): + """ + Text classification evaluator. + This text classification evaluator can currently be loaded from [`evaluator`] using the default task name + `text-classification` or with a `"sentiment-analysis"` alias. + Methods in this class assume a data format compatible with the [`~transformers.TextClassificationPipeline`] - a single textual + feature as input and a categorical label as output. + """ + + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="text-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def prepare_data(self, data: Union[str, Dataset], input_column: str, second_input_column: str, label_column: str): + if data is None: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + + self.check_required_columns(data, {"input_column": input_column, "label_column": label_column}) + + if second_input_column is not None: + self.check_required_columns(data, {"second_input_column": second_input_column}) + + data = load_dataset(data) if isinstance(data, str) else data + + return {"references": data[label_column]}, DatasetColumnPair( + data, input_column, second_input_column, "text", "text_pair" + ) + + def predictions_processor(self, predictions, label_mapping): + predictions = [ + label_mapping[element["label"]] if label_mapping is not None else element["label"] + for element in predictions + ] + return {"predictions": predictions} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + second_input_column: Optional[str] = None, + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, *optional*, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + second_input_column (`str`, *optional*, defaults to `None`): + The name of the second column containing the text features. This may be useful for classification tasks + as MNLI, where two columns are used. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, input_column=input_column, second_input_column=second_input_column, label_column=label_column + ) + pipe = self.prepare_pipeline( + model_or_pipeline=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, label_mapping) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/text_generation.py b/ST/evaluate/src/evaluate/evaluator/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..15522e860f7eb6fc693780f637337c0fdb22a21c --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/text_generation.py @@ -0,0 +1,69 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple + +from datasets import Dataset + +from .base import Evaluator +from .utils import DatasetColumn + + +TASK_DOCUMENTATION_KWARGS = r""" + input_column (`str`, defaults to `"text"`): + the name of the column containing the input text in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. +""" + + +class TextGenerationEvaluator(Evaluator): + """ + Text generation evaluator. + This Text generation evaluator can currently be loaded from [`evaluator`] using the default task name + `text-generation`. + Methods in this class assume a data format compatible with the [`~transformers.TextGenerationPipeline`]. + """ + + def predictions_processor(self, predictions, *args, **kwargs): + """ + Args: + predictions: A list of lists of dicts + + Returns: + `dict`: All the generated texts are flattened and stored under the "data" key. + """ + return {"data": [pred[f"{self.predictions_prefix}_text"] for pred_list in predictions for pred in pred_list]} + + def __init__(self, task="text-generation", default_metric_name=None, predictions_prefix: str = "generated"): + super().__init__(task=task, default_metric_name=default_metric_name) + self.predictions_prefix = predictions_prefix + + def prepare_data(self, data: Dataset, input_column: str, *args, **kwargs) -> Tuple[Dict, DatasetColumn]: + """ + Prepare data. + + Args: + data ([`Dataset`]): + Specifies the dataset we will run evaluation on. + input_column (`str`, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + Returns: + `dict`: metric inputs. + `list`: pipeline inputs. + """ + + self.check_required_columns(data, {"input_column": input_column}) + + return {}, DatasetColumn(data, input_column) diff --git a/ST/evaluate/src/evaluate/evaluator/token_classification.py b/ST/evaluate/src/evaluate/evaluator/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..ba08ebd58d72417eed4e20c93a46c53adaa49811 --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/token_classification.py @@ -0,0 +1,278 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union + +from datasets import ClassLabel, Dataset, Sequence +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumn + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following [conll2003 dataset](https://huggingface.co/datasets/conll2003). Datasets whose inputs are single strings, and labels are a list of offset are not supported. + + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("token-classification") + >>> data = load_dataset("conll2003", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english", + >>> data=data, + >>> metric="seqeval", + >>> ) + ``` + + + + For example, the following dataset format is accepted by the evaluator: + + ```python + dataset = Dataset.from_dict( + mapping={ + "tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]], + "ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]], + }, + features=Features({ + "tokens": Sequence(feature=Value(dtype="string")), + "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])), + }), + ) + ``` + + + + + + For example, the following dataset format is **not** accepted by the evaluator: + + ```python + dataset = Dataset.from_dict( + mapping={ + "tokens": [["New York is a city and Felix a person."]], + "starts": [[0, 23]], + "ends": [[7, 27]], + "ner_tags": [["LOC", "PER"]], + }, + features=Features({ + "tokens": Value(dtype="string"), + "starts": Sequence(feature=Value(dtype="int32")), + "ends": Sequence(feature=Value(dtype="int32")), + "ner_tags": Sequence(feature=Value(dtype="string")), + }), + ) + ``` + + +""" + + +class TokenClassificationEvaluator(Evaluator): + """ + Token classification evaluator. + + This token classification evaluator can currently be loaded from [`evaluator`] using the default task name + `token-classification`. + + Methods in this class assume a data format compatible with the [`~transformers.TokenClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {"ignore_labels": []} + + def __init__(self, task="token-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions: List[List[Dict]], words: List[List[str]], join_by: str): + """ + Transform the pipeline predictions into a list of predicted labels of the same length as the true labels. + + Args: + predictions (`List[List[Dict]]`): + List of pipeline predictions, where each token has been labeled. + words (`List[List[str]]`): + Original input data to the pipeline, used to build predicted labels of the same length. + join_by (`str`): + String to use to join two words. In English, it will typically be " ". + + Returns: + `dict`: a dictionary holding the predictions + """ + preds = [] + + # iterate over the data rows + for i, prediction in enumerate(predictions): + pred_processed = [] + + # get a list of tuples giving the indexes of the start and end character of each word + words_offsets = self.words_to_offsets(words[i], join_by) + + token_index = 0 + for word_offset in words_offsets: + # for each word, we may keep only the predicted label for the first token, discard the others + while prediction[token_index]["start"] < word_offset[0]: + token_index += 1 + + if prediction[token_index]["start"] > word_offset[0]: # bad indexing + pred_processed.append("O") + elif prediction[token_index]["start"] == word_offset[0]: + pred_processed.append(prediction[token_index]["entity"]) + + preds.append(pred_processed) + + return {"predictions": preds} + + def words_to_offsets(self, words: List[str], join_by: str): + """ + Convert a list of words to a list of offsets, where word are joined by `join_by`. + + Args: + words (`List[str]`): + List of words to get offsets from. + join_by (`str`): + String to insert between words. + + Returns: + `List[Tuple[int, int]]`: List of the characters (start index, end index) for each of the words. + """ + offsets = [] + + start = 0 + for word in words: + end = start + len(word) - 1 + offsets.append((start, end)) + start = end + len(join_by) + 1 + + return offsets + + def prepare_data(self, data: Union[str, Dataset], input_column: str, label_column: str, join_by: str): + super().prepare_data(data, input_column, label_column) + + if not isinstance(data.features[input_column], Sequence) or not isinstance( + data.features[label_column], Sequence + ): + raise ValueError( + "TokenClassificationEvaluator expects the input and label columns to be provided as lists." + ) + + # If the labels are of type ClassLabel, they are already integers and we have the map stored somewhere. + # Otherwise, we have to get the list of labels manually. + labels_are_int = isinstance(data.features[label_column].feature, ClassLabel) + if labels_are_int: + label_list = data.features[label_column].feature.names # list of string labels + id_to_label = {i: label for i, label in enumerate(label_list)} + references = [[id_to_label[label_id] for label_id in label_ids] for label_ids in data[label_column]] + elif data.features[label_column].feature.dtype.startswith("int"): + raise NotImplementedError( + "References provided as integers, but the reference column is not a Sequence of ClassLabels." + ) + else: + # In the event the labels are not a `Sequence[ClassLabel]`, we have already labels as strings + # An example is labels as ["PER", "PER", "O", "LOC", "O", "LOC", "O"], e.g. in polyglot_ner dataset + references = data[label_column] + + metric_inputs = {"references": references} + data = data.map(lambda x: {input_column: join_by.join(x[input_column])}) + pipeline_inputs = DatasetColumn(data, input_column) + + return metric_inputs, pipeline_inputs + + def prepare_pipeline( + self, + model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"], # noqa: F821 + tokenizer: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + feature_extractor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + device: int = None, + ): + pipe = super().prepare_pipeline(model_or_pipeline, tokenizer, feature_extractor, device) + + # check the pipeline outputs start characters in its predictions + dummy_output = pipe(["2003 New York Gregory"], **self.PIPELINE_KWARGS) + if dummy_output[0][0]["start"] is None: + raise ValueError( + "TokenClassificationEvaluator supports only pipelines giving 'start' index as a pipeline output (got None). " + "Transformers pipelines with a slow tokenizer will raise this error." + ) + + return pipe + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: str = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: Optional[int] = None, + random_state: Optional[int] = None, + input_column: str = "tokens", + label_column: str = "ner_tags", + join_by: Optional[str] = " ", + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, defaults to `"tokens"`): + The name of the column containing the tokens feature in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + join_by (`str`, *optional*, defaults to `" "`): + This evaluator supports dataset whose input column is a list of words. This parameter specifies how to join + words to generate a string input. This is especially useful for languages that do not separate words by a space. + """ + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, input_column=input_column, label_column=label_column, join_by=join_by + ) + pipe = self.prepare_pipeline(model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, data[input_column], join_by) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/ST/evaluate/src/evaluate/evaluator/utils.py b/ST/evaluate/src/evaluate/evaluator/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e364276d008b689d726b8dbbea1402fa93886d9b --- /dev/null +++ b/ST/evaluate/src/evaluate/evaluator/utils.py @@ -0,0 +1,84 @@ +from datasets import Dataset, get_dataset_split_names + + +class DatasetColumn(list): + """Helper class to avoid loading a dataset column into memory when accessing it.""" + + def __init__(self, dataset: Dataset, key: str): + self.dataset = dataset + self.key = key + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, i): + return self.dataset[i][self.key] + + def __iter__(self): + return (self.dataset[i][self.key] for i in range(len(self))) + + +def choose_split(data, subset=None): + available_splits = get_dataset_split_names(data, subset) + preferred_split_order = [ + "test", + "testing", + "eval", + "evaluation", + "validation", + "val", + "valid", + "dev", + "train", + "training", + ] + for split in preferred_split_order: + if split in available_splits: + return split + raise ValueError("No dataset split defined! Pass an explicit value to the `split` kwarg.") + + +class DatasetColumnPair(list): + """Helper class to avoid loading two dataset columns into memory when accessing it.""" + + def __init__( + self, + dataset: Dataset, + first_col: str, + second_col: str, + first_key: str, + second_key: str, + ): + """ + Args: + dataset (Dataset): dataset to build an iterator on + first_col (str): first column name to use in the dataset + second_col (str): second column name to use in the dataset + first_key (str): key name used for the first column in the returned dictionary + second_key (str): key name used for the second column in the returned dictionary + """ + self.dataset = dataset + + self.first_col = first_col + self.second_col = second_col + + self.first_key = first_key + self.second_key = second_key + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, i): + return { + self.first_key: self.dataset[i][self.first_col], + self.second_key: self.dataset[i][self.second_col] if self.second_col else None, + } + + def __iter__(self): + return ( + { + self.first_key: self.dataset[i][self.first_col], + self.second_key: self.dataset[i][self.second_col] if self.second_col else None, + } + for i in range(len(self)) + ) diff --git a/ST/evaluate/src/evaluate/hub.py b/ST/evaluate/src/evaluate/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..86118332c6d2293f475e84e80726364dcc63e292 --- /dev/null +++ b/ST/evaluate/src/evaluate/hub.py @@ -0,0 +1,133 @@ +from typing import Dict + +import requests +from huggingface_hub import dataset_info, model_info +from huggingface_hub.repocard import metadata_update + +from .config import HF_HUB_ALLOWED_TASKS +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +def push_to_hub( + model_id: str, + task_type: str, + dataset_type: str, + dataset_name: str, + metric_type: str, + metric_name: str, + metric_value: float, + task_name: str = None, + dataset_config: str = None, + dataset_split: str = None, + dataset_revision: str = None, + dataset_args: Dict[str, int] = None, + metric_config: str = None, + metric_args: Dict[str, int] = None, + overwrite: bool = False, +): + r""" + Pushes the result of a metric to the metadata of a model repository in the Hub. + + Args: + model_id (`str`): + Model id from https://hf.co/models. + task_type (`str`): + Task id, refer to the [Hub allowed tasks](https://github.com/huggingface/evaluate/blob/main/src/evaluate/config.py#L154) for allowed values. + dataset_type (`str`): + Dataset id from https://hf.co/datasets. + dataset_name (`str`): + Pretty name for the dataset. + metric_type (`str`): + Metric id from https://hf.co/metrics. + metric_name (`str`): + Pretty name for the metric. + metric_value (`float`): + Computed metric value. + task_name (`str`, *optional*): + Pretty name for the task. + dataset_config (`str`, *optional*): + Dataset configuration used in [`~datasets.load_dataset`]. + See [`~datasets.load_dataset`] for more info. + dataset_split (`str`, *optional*): + Name of split used for metric computation. + dataset_revision (`str`, *optional*): + Git hash for the specific version of the dataset. + dataset_args (`dict[str, int]`, *optional*): + Additional arguments passed to [`~datasets.load_dataset`]. + metric_config (`str`, *optional*): + Configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + metric_args (`dict[str, int]`, *optional*): + Arguments passed during [`~evaluate.EvaluationModule.compute`]. + overwrite (`bool`, *optional*, defaults to `False`): + If set to `True` an existing metric field can be overwritten, otherwise + attempting to overwrite any existing fields will cause an error. + + Example: + + ```python + >>> push_to_hub( + ... model_id="huggingface/gpt2-wikitext2", + ... metric_value=0.5 + ... metric_type="bleu", + ... metric_name="BLEU", + ... dataset_name="WikiText", + ... dataset_type="wikitext", + ... dataset_split="test", + ... task_type="text-generation", + ... task_name="Text Generation" + ... ) + ```""" + if task_type not in HF_HUB_ALLOWED_TASKS: + raise ValueError(f"Task type not supported. Task has to be one of {HF_HUB_ALLOWED_TASKS}") + + try: + dataset_info(dataset_type) + except requests.exceptions.HTTPError: + logger.warning(f"Dataset {dataset_type} not found on the Hub at hf.co/datasets/{dataset_type}") + + try: + model_info(model_id) + except requests.exceptions.HTTPError: + raise ValueError(f"Model {model_id} not found on the Hub at hf.co/{model_id}") + + result = { + "task": { + "type": task_type, + }, + "dataset": { + "type": dataset_type, + "name": dataset_name, + }, + "metrics": [ + { + "type": metric_type, + "value": metric_value, + }, + ], + } + + if dataset_config is not None: + result["dataset"]["config"] = dataset_config + if dataset_split is not None: + result["dataset"]["split"] = dataset_split + if dataset_revision is not None: + result["dataset"]["revision"] = dataset_revision + if dataset_args is not None: + result["dataset"]["args"] = dataset_args + + if task_name is not None: + result["task"]["name"] = task_name + + if metric_name is not None: + result["metrics"][0]["name"] = metric_name + if metric_config is not None: + result["metrics"][0]["config"] = metric_config + if metric_args is not None: + result["metrics"][0]["args"] = metric_args + + metadata = {"model-index": [{"results": [result]}]} + + return metadata_update(repo_id=model_id, metadata=metadata, overwrite=overwrite) diff --git a/ST/evaluate/src/evaluate/info.py b/ST/evaluate/src/evaluate/info.py new file mode 100644 index 0000000000000000000000000000000000000000..cc095784e4f1c1f473dd85955447d97d5fdc4e65 --- /dev/null +++ b/ST/evaluate/src/evaluate/info.py @@ -0,0 +1,157 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModuleInfo records information we know about a dataset and a metric. +""" + +import dataclasses +import json +import os +from dataclasses import asdict, dataclass, field +from typing import List, Optional, Union + +from datasets.features import Features, Value + +from . import config +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +@dataclass +class EvaluationModuleInfo: + """Base class to store information about an evaluation used for `MetricInfo`, `ComparisonInfo`, + and `MeasurementInfo`. + + `EvaluationModuleInfo` documents an evaluation, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + # Set in the dataset scripts + description: str + citation: str + features: Union[Features, List[Features]] + inputs_description: str = field(default_factory=str) + homepage: str = field(default_factory=str) + license: str = field(default_factory=str) + codebase_urls: List[str] = field(default_factory=list) + reference_urls: List[str] = field(default_factory=list) + streamable: bool = False + format: Optional[str] = None + module_type: str = "metric" # deprecate this in the future + + # Set later by the builder + module_name: Optional[str] = None + config_name: Optional[str] = None + experiment_id: Optional[str] = None + + def __post_init__(self): + if self.format is not None: + for key, value in self.features.items(): + if not isinstance(value, Value): + raise ValueError( + f"When using 'numpy' format, all features should be a `datasets.Value` feature. " + f"Here {key} is an instance of {value.__class__.__name__}" + ) + + def write_to_directory(self, metric_info_dir): + """Write `EvaluationModuleInfo` as JSON to `metric_info_dir`. + Also save the license separately in LICENSE. + + Args: + metric_info_dir (`str`): + The directory to save `metric_info_dir` to. + + Example: + + ```py + >>> my_metric.info.write_to_directory("/path/to/directory/") + ``` + """ + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), "w", encoding="utf-8") as f: + json.dump(asdict(self), f) + + with open(os.path.join(metric_info_dir, config.LICENSE_FILENAME), "w", encoding="utf-8") as f: + f.write(self.license) + + @classmethod + def from_directory(cls, metric_info_dir) -> "EvaluationModuleInfo": + """Create `EvaluationModuleInfo` from the JSON file in `metric_info_dir`. + + Args: + metric_info_dir (`str`): + The directory containing the `metric_info` JSON file. This + should be the root directory of a specific metric version. + + Example: + + ```py + >>> my_metric = EvaluationModuleInfo.from_directory("/path/to/directory/") + ``` + """ + logger.info(f"Loading Metric info from {metric_info_dir}") + if not metric_info_dir: + raise ValueError("Calling EvaluationModuleInfo.from_directory() with undefined metric_info_dir.") + + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), encoding="utf-8") as f: + metric_info_dict = json.load(f) + return cls.from_dict(metric_info_dict) + + @classmethod + def from_dict(cls, metric_info_dict: dict) -> "EvaluationModuleInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in metric_info_dict.items() if k in field_names}) + + +@dataclass +class MetricInfo(EvaluationModuleInfo): + """Information about a metric. + + `EvaluationModuleInfo` documents a metric, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "metric" + + +@dataclass +class ComparisonInfo(EvaluationModuleInfo): + """Information about a comparison. + + `EvaluationModuleInfo` documents a comparison, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "comparison" + + +@dataclass +class MeasurementInfo(EvaluationModuleInfo): + """Information about a measurement. + + `EvaluationModuleInfo` documents a measurement, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "measurement" diff --git a/ST/evaluate/src/evaluate/inspect.py b/ST/evaluate/src/evaluate/inspect.py new file mode 100644 index 0000000000000000000000000000000000000000..20e2af28ed4df4e99c6d67cccdd24dda1c8cecf9 --- /dev/null +++ b/ST/evaluate/src/evaluate/inspect.py @@ -0,0 +1,129 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" List and inspect metrics.""" + +from typing import Optional + +import requests +from datasets import DownloadConfig + +from .config import EVALUATION_MODULE_TYPES, HF_LIST_ENDPOINT +from .loading import evaluation_module_factory +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +class SplitsNotFoundError(ValueError): + pass + + +def list_evaluation_modules(module_type=None, include_community=True, with_details=False): + """List all evaluation modules available on the Hugging Face Hub. + + Args: + module_type (`str`, *optional*, defaults to `None`): + Type of evaluation modules to list. Has to be one of `'metric'`, `'comparison'`, or `'measurement'`. If `None`, all types are listed. + include_community (`bool`, *optional*, defaults to `True`): + Include community modules in the list. + with_details (`bool`, *optional*, defaults to `False`): + Return the full details on the metrics instead of only the ID. + + Returns: + `List[Union[str, dict]]` + + Example: + + ```py + >>> from evaluate import list_evaluation_modules + >>> list_evaluation_modules(module_type="metric") + ``` + """ + + if module_type is None: + evaluations_list = [] + for module_type in EVALUATION_MODULE_TYPES: + evaluations_list.extend( + _list_evaluation_modules_type( + module_type, include_community=include_community, with_details=with_details + ) + ) + else: + if module_type not in EVALUATION_MODULE_TYPES: + raise ValueError(f"Invalid module type '{module_type}'. Has to be one of {EVALUATION_MODULE_TYPES}.") + evaluations_list = _list_evaluation_modules_type( + module_type, include_community=include_community, with_details=with_details + ) + return evaluations_list + + +def _list_evaluation_modules_type(module_type, include_community=True, with_details=False): + + r = requests.get(HF_LIST_ENDPOINT.format(type=module_type)) + r.raise_for_status() + d = r.json() + + if not include_community: + d = [element for element in d if element["id"].split("/")[0] == f"evaluate-{module_type}"] + + # remove namespace for canonical modules and add community tag + for element in d: + if element["id"].split("/")[0] == f"evaluate-{module_type}": + element["id"] = element["id"].split("/")[1] + element["community"] = False + else: + element["community"] = True + + if with_details: + return [ + { + "name": element["id"], + "type": module_type, + "community": element["community"], + "likes": element.get("likes", 0), + } + for element in d + ] + else: + return [element["id"] for element in d] + + +def inspect_evaluation_module( + path: str, local_path: str, download_config: Optional[DownloadConfig] = None, **download_kwargs +): + r""" + Allow inspection/modification of a evaluation script by copying it on local drive at local_path. + + Args: + path (``str``): path to the evaluation script. Can be either: + + - a local path to script or the directory containing the script (if the script has the same name as the directory), + e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'`` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``evaluate.list_evaluation_modules()``) + e.g. ``'accuracy'``, ``'bleu'`` or ``'word_length'`` + local_path (``str``): path to the local folder to copy the datset script to. + download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters. + **download_kwargs: optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. + """ + evaluation_module = evaluation_module_factory( + path, download_config=download_config, force_local_path=local_path, **download_kwargs + ) + print( + f"The processing scripts for metric {path} can be inspected at {local_path}. " + f"The main class is in {evaluation_module.module_path}. " + f"You can modify this processing scripts and use it with `evaluate.load({local_path})`." + ) diff --git a/ST/evaluate/src/evaluate/loading.py b/ST/evaluate/src/evaluate/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..43b76faf3babee6a36cf0a53727302035d99319a --- /dev/null +++ b/ST/evaluate/src/evaluate/loading.py @@ -0,0 +1,770 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Access datasets.""" +import filecmp +import importlib +import inspect +import json +import os +import re +import shutil +import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple, Type, Union +from urllib.parse import urlparse + +from datasets import DownloadConfig, DownloadMode +from datasets.builder import DatasetBuilder +from datasets.packaged_modules import _EXTENSION_TO_MODULE, _hash_python_lines +from datasets.utils.filelock import FileLock +from datasets.utils.version import Version + +from . import SCRIPTS_VERSION, config +from .module import EvaluationModule +from .utils.file_utils import ( + cached_path, + head_hf_s3, + hf_hub_url, + init_hf_modules, + is_relative_path, + relative_to_absolute_path, + url_or_path_join, +) +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +ALL_ALLOWED_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + ["zip"] + + +def init_dynamic_modules( + name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, hf_modules_cache: Optional[Union[Path, str]] = None +): + """ + Create a module with name `name` in which you can add dynamic modules + such as metrics or datasets. The module can be imported using its name. + The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can + be overriden by specifying a path to another directory in `hf_modules_cache`. + """ + hf_modules_cache = init_hf_modules(hf_modules_cache) + dynamic_modules_path = os.path.join(hf_modules_cache, name) + os.makedirs(dynamic_modules_path, exist_ok=True) + if not os.path.exists(os.path.join(dynamic_modules_path, "__init__.py")): + with open(os.path.join(dynamic_modules_path, "__init__.py"), "w"): + pass + return dynamic_modules_path + + +def import_main_class(module_path) -> Optional[Union[Type[DatasetBuilder], Type[EvaluationModule]]]: + """Import a module at module_path and return its main class, a Metric by default""" + module = importlib.import_module(module_path) + main_cls_type = EvaluationModule + + # Find the main class in our imported module + module_main_cls = None + for name, obj in module.__dict__.items(): + if isinstance(obj, type) and issubclass(obj, main_cls_type): + if inspect.isabstract(obj): + continue + module_main_cls = obj + break + + return module_main_cls + + +def files_to_hash(file_paths: List[str]) -> str: + """ + Convert a list of scripts or text files provided in file_paths into a hashed filename in a repeatable way. + """ + # List all python files in directories if directories are supplied as part of external imports + to_use_files: List[Union[Path, str]] = [] + for file_path in file_paths: + if os.path.isdir(file_path): + to_use_files.extend(list(Path(file_path).rglob("*.[pP][yY]"))) + else: + to_use_files.append(file_path) + + # Get the code from all these files + lines = [] + for file_path in to_use_files: + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + return _hash_python_lines(lines) + + +def convert_github_url(url_path: str) -> Tuple[str, Optional[str]]: + """Convert a link to a file on a github repo in a link to the raw github object.""" + parsed = urlparse(url_path) + sub_directory = None + if parsed.scheme in ("http", "https", "s3") and parsed.netloc == "github.com": + if "blob" in url_path: + if not url_path.endswith(".py"): + raise ValueError(f"External import from github at {url_path} should point to a file ending with '.py'") + url_path = url_path.replace("blob", "raw") # Point to the raw file + else: + # Parse github url to point to zip + github_path = parsed.path[1:] + repo_info, branch = github_path.split("/tree/") if "/tree/" in github_path else (github_path, "master") + repo_owner, repo_name = repo_info.split("/") + url_path = f"https://github.com/{repo_owner}/{repo_name}/archive/{branch}.zip" + sub_directory = f"{repo_name}-{branch}" + return url_path, sub_directory + + +def increase_load_count(name: str, resource_type: str): + """Update the download count of a dataset or metric.""" + if not config.HF_EVALUATE_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: + try: + head_hf_s3(name, filename=name + ".py", dataset=(resource_type == "dataset")) + except Exception: + pass + + +def get_imports(file_path: str) -> Tuple[str, str, str, str]: + """Find whether we should import or clone additional files for a given processing script. + And list the import. + + We allow: + - library dependencies, + - local dependencies and + - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. + external dependencies will be downloaded (and extracted if needed in the dataset folder). + We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. + + Note that only direct import in the dataset processing script will be handled + We don't recursively explore the additional import to download further files. + + Example:: + + import tensorflow + import .c4_utils + import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset + """ + lines = [] + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + + logger.debug(f"Checking {file_path} for additional imports.") + imports: List[Tuple[str, str, str, Optional[str]]] = [] + is_in_docstring = False + for line in lines: + docstr_start_match = re.findall(r'[\s\S]*?"""[\s\S]*?', line) + + if len(docstr_start_match) == 1: + # flip True <=> False only if doctstring + # starts at line without finishing + is_in_docstring = not is_in_docstring + + if is_in_docstring: + # import statements in doctstrings should + # not be added as required dependencies + continue + + match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + if match is None: + match = re.match( + r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) + if match is None: + continue + if match.group(1): + # The import starts with a '.', we will download the relevant file + if any(imp[1] == match.group(2) for imp in imports): + # We already have this import + continue + if match.group(3): + # The import has a comment with 'From:', we'll retrieve it from the given url + url_path = match.group(3) + url_path, sub_directory = convert_github_url(url_path) + imports.append(("external", match.group(2), url_path, sub_directory)) + elif match.group(2): + # The import should be at the same place as the file + imports.append(("internal", match.group(2), match.group(2), None)) + else: + if match.group(3): + # The import has a comment with `From: git+https:...`, asks user to pip install from git. + url_path = match.group(3) + imports.append(("library", match.group(2), url_path, None)) + else: + imports.append(("library", match.group(2), match.group(2), None)) + + return imports + + +def _download_additional_modules( + name: str, base_path: str, imports: Tuple[str, str, str, str], download_config: Optional[DownloadConfig] +) -> List[Tuple[str, str]]: + """ + Download additional module for a module .py at URL (or local path) /.py + The imports must have been parsed first using ``get_imports``. + + If some modules need to be installed with pip, an error is raised showing how to install them. + This function return the list of downloaded modules as tuples (import_name, module_file_path). + + The downloaded modules can then be moved into an importable directory with ``_copy_script_and_other_resources_in_importable_dir``. + """ + local_imports = [] + library_imports = [] + download_config = download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading extra modules" + for import_type, import_name, import_path, sub_directory in imports: + if import_type == "library": + library_imports.append((import_name, import_path)) # Import from a library + continue + + if import_name == name: + raise ValueError( + f"Error in the {name} script, importing relative {import_name} module " + f"but {import_name} is the name of the script. " + f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' " + f"comment pointing to the original relative import file path." + ) + if import_type == "internal": + url_or_filename = url_or_path_join(base_path, import_path + ".py") + elif import_type == "external": + url_or_filename = import_path + else: + raise ValueError("Wrong import_type") + + local_import_path = cached_path( + url_or_filename, + download_config=download_config, + ) + if sub_directory is not None: + local_import_path = os.path.join(local_import_path, sub_directory) + local_imports.append((import_name, local_import_path)) + + # Check library imports + needs_to_be_installed = set() + for library_import_name, library_import_path in library_imports: + try: + lib = importlib.import_module(library_import_name) # noqa F841 + except ImportError: + needs_to_be_installed.add((library_import_name, library_import_path)) + if needs_to_be_installed: + raise ImportError( + f"To be able to use {name}, you need to install the following dependencies" + f"{[lib_name for lib_name, lib_path in needs_to_be_installed]} using 'pip install " + f"{' '.join([lib_path for lib_name, lib_path in needs_to_be_installed])}' for instance'" + ) + return local_imports + + +def _copy_script_and_other_resources_in_importable_dir( + name: str, + importable_directory_path: str, + subdirectory_name: str, + original_local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + download_mode: Optional[DownloadMode], +) -> str: + """Copy a script and its required imports to an importable directory + + Args: + name (str): name of the resource to load + importable_directory_path (str): path to the loadable folder in the dynamic modules directory + subdirectory_name (str): name of the subdirectory in importable_directory_path in which to place the script + original_local_path (str): local path to the resource script + local_imports (List[Tuple[str, str]]): list of (destination_filename, import_file_to_copy) + additional_files (List[Tuple[str, str]]): list of (destination_filename, additional_file_to_copy) + download_mode (Optional[DownloadMode]): download mode + + Return: + importable_local_file: path to an importable module with importlib.import_module + """ + + # Define a directory with a unique name in our dataset or metric folder + # path is: ./datasets|metrics/dataset|metric_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) + importable_local_file = os.path.join(importable_subdirectory, name + ".py") + + # Prevent parallel disk operations + lock_path = importable_directory_path + ".lock" + with FileLock(lock_path): + # Create main dataset/metrics folder if needed + if download_mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(importable_directory_path): + shutil.rmtree(importable_directory_path) + os.makedirs(importable_directory_path, exist_ok=True) + + # add an __init__ file to the main dataset folder if needed + init_file_path = os.path.join(importable_directory_path, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Create hash dataset folder if needed + os.makedirs(importable_subdirectory, exist_ok=True) + # add an __init__ file to the hash dataset folder if needed + init_file_path = os.path.join(importable_subdirectory, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Copy dataset.py file in hash folder if needed + if not os.path.exists(importable_local_file): + shutil.copyfile(original_local_path, importable_local_file) + + # Record metadata associating original dataset path with local unique folder + meta_path = importable_local_file.split(".py")[0] + ".json" + if not os.path.exists(meta_path): + meta = {"original file path": original_local_path, "local file path": importable_local_file} + # the filename is *.py in our case, so better rename to filenam.json instead of filename.py.json + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + # Copy all the additional imports + for import_name, import_path in local_imports: + if os.path.isfile(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name + ".py") + if not os.path.exists(full_path_local_import): + shutil.copyfile(import_path, full_path_local_import) + elif os.path.isdir(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name) + if not os.path.exists(full_path_local_import): + shutil.copytree(import_path, full_path_local_import) + else: + raise OSError(f"Error with local import at {import_path}") + + # Copy aditional files like dataset infos file if needed + for file_name, original_path in additional_files: + destination_additional_path = os.path.join(importable_subdirectory, file_name) + if not os.path.exists(destination_additional_path) or not filecmp.cmp( + original_path, destination_additional_path + ): + shutil.copyfile(original_path, destination_additional_path) + return importable_local_file + + +def _create_importable_file( + local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + dynamic_modules_path: str, + module_namespace: str, + name: str, + download_mode: DownloadMode, +) -> Tuple[str, str]: + importable_directory_path = os.path.join(dynamic_modules_path, module_namespace, name.replace("/", "--")) + Path(importable_directory_path).mkdir(parents=True, exist_ok=True) + (Path(importable_directory_path).parent / "__init__.py").touch(exist_ok=True) + hash = files_to_hash([local_path] + [loc[1] for loc in local_imports]) + importable_local_file = _copy_script_and_other_resources_in_importable_dir( + name=name.split("/")[-1], + importable_directory_path=importable_directory_path, + subdirectory_name=hash, + original_local_path=local_path, + local_imports=local_imports, + additional_files=additional_files, + download_mode=download_mode, + ) + logger.debug(f"Created importable dataset file at {importable_local_file}") + module_path = ".".join( + [os.path.basename(dynamic_modules_path), module_namespace, name.replace("/", "--"), hash, name.split("/")[-1]] + ) + return module_path, hash + + +@dataclass +class ImportableModule: + module_path: str + hash: str + + +class _EvaluationModuleFactory: + def get_module(self) -> ImportableModule: + raise NotImplementedError + + +class LocalEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a local metric. The metric script is loaded from a local script.""" + + def __init__( + self, + path: str, + module_type: str = "metrics", + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.path = path + self.module_type = module_type + self.name = Path(path).stem + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + + def get_module(self) -> ImportableModule: + # get script and other files + imports = get_imports(self.path) + local_imports = _download_additional_modules( + name=self.name, + base_path=str(Path(self.path).parent), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=self.path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class HubEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a metric from a metric repository on the Hub.""" + + def __init__( + self, + name: str, + module_type: str = "metrics", + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.revision = revision + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 1 + increase_load_count(name, resource_type="metric") + + def download_loading_script(self, revision) -> str: + file_path = hf_hub_url(path=self.name, name=self.name.split("/")[1] + ".py", revision=revision) + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading builder script" + return cached_path(file_path, download_config=download_config) + + def get_module(self) -> ImportableModule: + revision = self.revision or os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) + + if re.match(r"\d*\.\d*\.\d*", revision): # revision is version number (three digits separated by full stops) + revision = "v" + revision # tagging convention on evaluate repository starts with v + + # get script and other files + try: + local_path = self.download_loading_script(revision) + except FileNotFoundError as err: + # if there is no file found with current revision tag try to load main + if self.revision is None and os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) != "main": + revision = "main" + local_path = self.download_loading_script(revision) + else: + raise err + + imports = get_imports(local_path) + local_imports = _download_additional_modules( + name=self.name, + base_path=hf_hub_url(path=self.name, name="", revision=revision), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=local_path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class CachedEvaluationModuleFactory(_EvaluationModuleFactory): + """ + Get the module of a metric that has been loaded once already and cached. + The script that is loaded from the cache is the most recent one with a matching name. + """ + + def __init__( + self, + name: str, + module_type: str = "metrics", + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 0 + + def get_module(self) -> ImportableModule: + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + importable_directory_path = os.path.join(dynamic_modules_path, self.module_type, self.name) + hashes = ( + [h for h in os.listdir(importable_directory_path) if len(h) == 64] + if os.path.isdir(importable_directory_path) + else None + ) + if not hashes: + raise FileNotFoundError(f"Metric {self.name} is not cached in {dynamic_modules_path}") + # get most recent + + def _get_modification_time(module_hash): + return ( + (Path(importable_directory_path) / module_hash / (self.name.split("--")[-1] + ".py")).stat().st_mtime + ) + + hash = sorted(hashes, key=_get_modification_time)[-1] + logger.warning( + f"Using the latest cached version of the module from {os.path.join(importable_directory_path, hash)} " + f"(last modified on {time.ctime(_get_modification_time(hash))}) since it " + f"couldn't be found locally at {self.name}, or remotely on the Hugging Face Hub." + ) + # make the new module to be noticed by the import system + module_path = ".".join( + [os.path.basename(dynamic_modules_path), self.module_type, self.name, hash, self.name.split("--")[-1]] + ) + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +def evaluation_module_factory( + path: str, + module_type: Optional[str] = None, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + force_local_path: Optional[str] = None, + dynamic_modules_path: Optional[str] = None, + **download_kwargs, +) -> ImportableModule: + """ + Download/extract/cache a metric module. + + Metrics codes are cached inside the the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). + + Args: + + path (str): Path or name of the metric script. + + - if ``path`` is a local metric script or a directory containing a local metric script (if the script has the same name as the directory): + -> load the module from the metric script + e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'``. + - if ``path`` is a metric on the Hugging Face Hub (ex: `glue`, `squad`) + -> load the module from the metric script in the github repository at huggingface/datasets + e.g. ``'accuracy'`` or ``'rouge'``. + + revision (Optional ``Union[str, datasets.Version]``): + If specified, the module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the master branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + force_local_path (Optional str): Optional path to a local path to download and prepare the script to. + Used to inspect or modify the script folder. + dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. + By default the datasets and metrics are stored inside the `datasets_modules` module. + download_kwargs: optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. + + Returns: + ImportableModule + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + download_config.extract_compressed_file = True + download_config.force_extract = True + + filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] + if not filename.endswith(".py"): + filename = filename + ".py" + combined_path = os.path.join(path, filename) + # Try locally + if path.endswith(filename): + if os.path.isfile(path): + return LocalEvaluationModuleFactory( + path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + else: + raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}") + elif os.path.isfile(combined_path): + return LocalEvaluationModuleFactory( + combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + elif is_relative_path(path) and path.count("/") <= 1 and not force_local_path: + try: + # load a canonical evaluation module from hub + if path.count("/") == 0: + # if no type provided look through all possible modules + if module_type is None: + for current_type in ["metric", "comparison", "measurement"]: + try: + return HubEvaluationModuleFactory( + f"evaluate-{current_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except ConnectionError: + pass + raise FileNotFoundError + # if module_type provided load specific module_type + else: + return HubEvaluationModuleFactory( + f"evaluate-{module_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + # load community evaluation module from hub + elif path.count("/") == 1: + return HubEvaluationModuleFactory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except Exception as e1: # noqa: all the attempts failed, before raising the error we should check if the module is already cached. + # if it's a canonical module we need to check if it's any of the types + if path.count("/") == 0: + for current_type in ["metric", "comparison", "measurement"]: + try: + return CachedEvaluationModuleFactory( + f"evaluate-{current_type}--{path}", dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + # if it's a community module we just need to check on path + elif path.count("/") == 1: + try: + return CachedEvaluationModuleFactory( + path.replace("/", "--"), dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + if not isinstance(e1, (ConnectionError, FileNotFoundError)): + raise e1 from None + raise FileNotFoundError( + f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}. " + f"Module '{path}' doesn't exist on the Hugging Face Hub either." + ) from None + else: + raise FileNotFoundError(f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}.") + + +def load( + path: str, + config_name: Optional[str] = None, + module_type: Optional[str] = None, + process_id: int = 0, + num_process: int = 1, + cache_dir: Optional[str] = None, + experiment_id: Optional[str] = None, + keep_in_memory: bool = False, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + revision: Optional[Union[str, Version]] = None, + **init_kwargs, +) -> EvaluationModule: + """Load a [`~evaluate.EvaluationModule`]. + + Args: + + path (`str`): + Path to the evaluation processing script with the evaluation builder. Can be either: + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./metrics/rouge'` or `'./metrics/rouge/rouge.py'` + - a evaluation module identifier on the HuggingFace evaluate repo e.g. `'rouge'` or `'bleu'` that are in either `'metrics/'`, + `'comparisons/'`, or `'measurements/'` depending on the provided `module_type` + config_name (`str`, *optional*): + Selecting a configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + module_type (`str`, default `'metric'`): + Type of evaluation module, can be one of `'metric'`, `'comparison'`, or `'measurement'`. + process_id (`int`, *optional*): + For distributed evaluation: id of the process. + num_process (`int`, *optional*): + For distributed evaluation: total number of processes. + cache_dir (`str`, *optional*): + Path to store the temporary predictions and references (default to `~/.cache/huggingface/evaluate/`). + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + keep_in_memory (`bool`): + Whether to store the temporary results in memory (defaults to `False`). + download_config ([`~evaluate.DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`], defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision (`Union[str, evaluate.Version]`, *optional*): + If specified, the module will be loaded from the datasets repository + at this version. By default it is set to the local version of the lib. Specifying a version that is different from + your local version of the lib might cause compatibility issues. + + Returns: + [`evaluate.EvaluationModule`] + + Example: + + ```py + >>> from evaluate import load + >>> accuracy = evaluate.load("accuracy") + ``` + """ + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + evaluation_module = evaluation_module_factory( + path, module_type=module_type, revision=revision, download_config=download_config, download_mode=download_mode + ) + evaluation_cls = import_main_class(evaluation_module.module_path) + evaluation_instance = evaluation_cls( + config_name=config_name, + process_id=process_id, + num_process=num_process, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + experiment_id=experiment_id, + hash=evaluation_module.hash, + **init_kwargs, + ) + + if module_type and module_type != evaluation_instance.module_type: + raise TypeError( + f"No module of module type '{module_type}' not found for '{path}' locally, or on the Hugging Face Hub. Found module of module type '{evaluation_instance.module_type}' instead." + ) + + # Download and prepare resources for the metric + evaluation_instance.download_and_prepare(download_config=download_config) + + return evaluation_instance diff --git a/ST/evaluate/src/evaluate/module.py b/ST/evaluate/src/evaluate/module.py new file mode 100644 index 0000000000000000000000000000000000000000..3652ad1b6ea0691afba35b70b3197b307a66a428 --- /dev/null +++ b/ST/evaluate/src/evaluate/module.py @@ -0,0 +1,1029 @@ +# Copyright 2020 The HuggingFace Datasets Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModule base class.""" +import collections +import itertools +import os +import types +import uuid +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa +from datasets import DatasetInfo, DownloadConfig, DownloadManager +from datasets.arrow_dataset import Dataset +from datasets.arrow_reader import ArrowReader +from datasets.arrow_writer import ArrowWriter +from datasets.features import Features, Sequence, Value +from datasets.features.features import _check_non_null_non_empty_recursive +from datasets.utils.filelock import BaseFileLock, FileLock, Timeout +from datasets.utils.py_utils import copyfunc, temp_seed, zip_dict + +from . import config +from .info import EvaluationModuleInfo +from .naming import camelcase_to_snakecase +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +class FileFreeLock(BaseFileLock): + """Thread lock until a file **cannot** be locked""" + + def __init__(self, lock_file, *args, **kwargs): + self.filelock = FileLock(lock_file) + super().__init__(lock_file, *args, **kwargs) + + def _acquire(self): + try: + self.filelock.acquire(timeout=0.01, poll_intervall=0.02) # Try to lock once + except Timeout: + # We couldn't acquire the lock, the file is locked! + self._lock_file_fd = self.filelock.lock_file + else: + # We were able to acquire the lock, the file is not yet locked! + self.filelock.release() + self._lock_file_fd = None + + def _release(self): + self._lock_file_fd = None + + +# lists - summarize long lists similarly to NumPy +# arrays/tensors - let the frameworks control formatting +def summarize_if_long_list(obj): + if not type(obj) == list or len(obj) <= 6: + return f"{obj}" + + def format_chunk(chunk): + return ", ".join(repr(x) for x in chunk) + + return f"[{format_chunk(obj[:3])}, ..., {format_chunk(obj[-3:])}]" + + +class EvaluationModuleInfoMixin: + """This base class exposes some attributes of EvaluationModuleInfo + at the base level of the EvaluationModule for easy access. + """ + + def __init__(self, info: EvaluationModuleInfo): + self._module_info = info + + @property + def info(self): + """:class:`evaluate.EvaluationModuleInfo` object containing all the metadata in the evaluation module.""" + return self._module_info + + @property + def name(self) -> str: + return self._module_info.module_name + + @property + def experiment_id(self) -> Optional[str]: + return self._module_info.experiment_id + + @property + def description(self) -> str: + return self._module_info.description + + @property + def citation(self) -> str: + return self._module_info.citation + + @property + def features(self) -> Features: + return self._module_info.features + + @property + def inputs_description(self) -> str: + return self._module_info.inputs_description + + @property + def homepage(self) -> Optional[str]: + return self._module_info.homepage + + @property + def license(self) -> str: + return self._module_info.license + + @property + def codebase_urls(self) -> Optional[List[str]]: + return self._module_info.codebase_urls + + @property + def reference_urls(self) -> Optional[List[str]]: + return self._module_info.reference_urls + + @property + def streamable(self) -> bool: + return self._module_info.streamable + + @property + def format(self) -> Optional[str]: + return self._module_info.format + + @property + def module_type(self) -> str: + return self._module_info.module_type + + +class EvaluationModule(EvaluationModuleInfoMixin): + """A `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a module computation script and prevents the module's data + to be overridden when the module loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute module in distributed setups (in particular non-additive modules like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + seed (`int`, optional): + If specified, this will temporarily set numpy's random seed when [`~evaluate.EvaluationModule.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + hash (`str`): + Used to identify the evaluation module according to the hashed file contents. + max_concurrent_cache_files (`int`): + Max number of concurrent module cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + def __init__( + self, + config_name: Optional[str] = None, + keep_in_memory: bool = False, + cache_dir: Optional[str] = None, + num_process: int = 1, + process_id: int = 0, + seed: Optional[int] = None, + experiment_id: Optional[str] = None, + hash: str = None, + max_concurrent_cache_files: int = 10000, + timeout: Union[int, float] = 100, + **kwargs, + ): + # prepare info + self.config_name = config_name or "default" + info = self._info() + info.module_name = camelcase_to_snakecase(self.__class__.__name__) + info.config_name = self.config_name + info.experiment_id = experiment_id or "default_experiment" + EvaluationModuleInfoMixin.__init__(self, info) # For easy access on low level + + # Safety checks on num_process and process_id + if not isinstance(process_id, int) or process_id < 0: + raise ValueError("'process_id' should be a number greater than 0") + if not isinstance(num_process, int) or num_process <= process_id: + raise ValueError("'num_process' should be a number greater than process_id") + if keep_in_memory and num_process != 1: + raise ValueError("Using 'keep_in_memory' is not possible in distributed setting (num_process > 1).") + + self.num_process = num_process + self.process_id = process_id + self.max_concurrent_cache_files = max_concurrent_cache_files + + self.keep_in_memory = keep_in_memory + self._data_dir_root = os.path.expanduser(cache_dir or config.HF_METRICS_CACHE) + self.data_dir = self._build_data_dir() + if seed is None: + _, seed, pos, *_ = np.random.get_state() + self.seed: int = seed[pos] if pos < 624 else seed[0] + else: + self.seed: int = seed + self.timeout: Union[int, float] = timeout + + # Update 'compute' and 'add' docstring + # methods need to be copied otherwise it changes the docstrings of every instance + self.compute = types.MethodType(copyfunc(self.compute), self) + self.add_batch = types.MethodType(copyfunc(self.add_batch), self) + self.add = types.MethodType(copyfunc(self.add), self) + self.compute.__func__.__doc__ += self.info.inputs_description + self.add_batch.__func__.__doc__ += self.info.inputs_description + self.add.__func__.__doc__ += self.info.inputs_description + + # self.arrow_schema = pa.schema(field for field in self.info.features.type) + self.selected_feature_format = None + self.buf_writer = None + self.writer = None + self.writer_batch_size = None + self.data = None + + # This is the cache file we store our predictions/references in + # Keep it None for now so we can (cloud)pickle the object + self.cache_file_name = None + self.filelock = None + self.rendez_vous_lock = None + + # This is all the cache files on which we have a lock when we are in a distributed setting + self.file_paths = None + self.filelocks = None + + # This fingerprints the evaluation module according to the hashed contents of the module code + self._hash = hash + + def __len__(self): + """Return the number of examples (predictions or predictions/references pair) + currently stored in the evaluation module's cache. + """ + return 0 if self.writer is None else len(self.writer) + + def __repr__(self): + return ( + f'EvaluationModule(name: "{self.name}", module_type: "{self.module_type}", ' + f'features: {self.features}, usage: """{self.inputs_description}""", ' + f"stored examples: {len(self)})" + ) + + def _build_data_dir(self): + """Path of this evaluation module in cache_dir: + Will be: + self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ + If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. + """ + builder_data_dir = self._data_dir_root + builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name) + os.makedirs(builder_data_dir, exist_ok=True) + return builder_data_dir + + def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]: + """Create a new cache file. If the default cache file is used, we generated a new hash.""" + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow") + filelock = None + for i in range(self.max_concurrent_cache_files): + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=timeout) + except Timeout: + # If we have reached the max number of attempts or we are not allow to find a free name (distributed setup) + # We raise an error + if self.num_process != 1: + raise ValueError( + f"Error in _create_cache_file: another evaluation module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + if i == self.max_concurrent_cache_files - 1: + raise ValueError( + f"Cannot acquire lock, too many evaluation module instance are operating concurrently on this file system." + f"You should set a larger value of max_concurrent_cache_files when creating the evaluation module " + f"(current value is {self.max_concurrent_cache_files})." + ) from None + # In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name. + file_uuid = str(uuid.uuid4()) + file_path = os.path.join( + self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow" + ) + else: + break + + return file_path, filelock + + def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]: + """Get a lock on all the cache files in a distributed setup. + We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). + """ + if self.num_process == 1: + if self.cache_file_name is None: + raise ValueError( + "Evaluation module cache file doesn't exist. Please make sure that you call `add` or `add_batch` " + "at least once before calling `compute`." + ) + file_paths = [self.cache_file_name] + else: + file_paths = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow") + for process_id in range(self.num_process) + ] + + # Let's acquire a lock on each process files to be sure they are finished writing + filelocks = [] + for process_id, file_path in enumerate(file_paths): + if process_id == 0: # process 0 already has its lock file + filelocks.append(self.filelock) + else: + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Cannot acquire lock on cached file {file_path} for process {process_id}." + ) from None + else: + filelocks.append(filelock) + + return file_paths, filelocks + + def _check_all_processes_locks(self): + expected_lock_file_names = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock") + for process_id in range(self.num_process) + ] + for expected_lock_file_name in expected_lock_file_names: + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + + def _check_rendez_vous(self): + expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock") + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + rendez_vous_lock = FileLock(lock_file_name) + try: + rendez_vous_lock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.") from None + else: + rendez_vous_lock.release() + + def _finalize(self): + """Close all the writing process and load/gather the data + from all the nodes if main node or all_process is True. + """ + if self.writer is not None: + self.writer.finalize() + self.writer = None + # release the locks of the processes > 0 so that process 0 can lock them to read + delete the data + if self.filelock is not None and self.process_id > 0: + self.filelock.release() + + if self.keep_in_memory: + # Read the predictions and references + reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset.from_buffer(self.buf_writer.getvalue()) + + elif self.process_id == 0: + # Let's acquire a lock on each node files to be sure they are finished writing + file_paths, filelocks = self._get_all_cache_files() + + # Read the predictions and references + try: + reader = ArrowReader(path="", info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths])) + except FileNotFoundError: + raise ValueError( + "Error in finalize: another evaluation module instance is already using the local cache file. " + "Please specify an experiment_id to avoid collision between distributed evaluation module instances." + ) from None + + # Store file paths and locks and we will release/delete them after the computation. + self.file_paths = file_paths + self.filelocks = filelocks + + def compute(self, *, predictions=None, references=None, **kwargs) -> Optional[dict]: + """Compute the evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (optional): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 1]) + ``` + """ + all_kwargs = {"predictions": predictions, "references": references, **kwargs} + if predictions is None and references is None: + missing_kwargs = {k: None for k in self._feature_names() if k not in all_kwargs} + all_kwargs.update(missing_kwargs) + else: + missing_inputs = [k for k in self._feature_names() if k not in all_kwargs] + if missing_inputs: + raise ValueError( + f"Evaluation module inputs are missing: {missing_inputs}. All required inputs are {list(self._feature_names())}" + ) + inputs = {input_name: all_kwargs[input_name] for input_name in self._feature_names()} + compute_kwargs = {k: kwargs[k] for k in kwargs if k not in self._feature_names()} + + if any(v is not None for v in inputs.values()): + self.add_batch(**inputs) + self._finalize() + + self.cache_file_name = None + self.filelock = None + self.selected_feature_format = None + + if self.process_id == 0: + self.data.set_format(type=self.info.format) + + inputs = {input_name: self.data[input_name] for input_name in self._feature_names()} + with temp_seed(self.seed): + output = self._compute(**inputs, **compute_kwargs) + + if self.buf_writer is not None: + self.buf_writer = None + del self.data + self.data = None + else: + # Release locks and delete all the cache files. Process 0 is released last. + for filelock, file_path in reversed(list(zip(self.filelocks, self.file_paths))): + logger.info(f"Removing {file_path}") + del self.data + self.data = None + del self.writer + self.writer = None + os.remove(file_path) + filelock.release() + + return output + else: + return None + + def add_batch(self, *, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for the evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... accuracy.add_batch(references=refs, predictions=preds) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_batch(batch) + self._init_writer() + try: + for key, column in batch.items(): + if len(column) > 0: + self._enforce_nested_string_type(self.selected_feature_format[key], column[0]) + batch = self.selected_feature_format.encode_batch(batch) + self.writer.write_batch(batch) + except (pa.ArrowInvalid, TypeError): + if any(len(batch[c]) != len(next(iter(batch.values()))) for c in batch): + col0 = next(iter(batch)) + bad_col = [c for c in batch if len(batch[c]) != len(batch[col0])][0] + error_msg = ( + f"Mismatch in the number of {col0} ({len(batch[col0])}) and {bad_col} ({len(batch[bad_col])})" + ) + elif set(self.selected_feature_format) != {"references", "predictions"}: + error_msg = ( + f"Module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(batch[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + else: + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + f"Input predictions: {summarize_if_long_list(predictions)},\n" + f"Input references: {summarize_if_long_list(references)}" + ) + raise ValueError(error_msg) from None + + def add(self, *, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for the evaluation module's stack. + + Args: + prediction (`list/array/tensor`, *optional*): + Predictions. + reference (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.add(references=[0,1], predictions=[1,0]) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + example = {"predictions": prediction, "references": reference, **kwargs} + example = {input_name: example[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_example(example) + self._init_writer() + try: + self._enforce_nested_string_type(self.selected_feature_format, example) + example = self.selected_feature_format.encode_example(example) + self.writer.write(example) + except (pa.ArrowInvalid, TypeError): + error_msg = ( + f"Evaluation module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format},\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(example[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + raise ValueError(error_msg) from None + + def _infer_feature_from_batch(self, batch): + if isinstance(self.features, Features): + return self.features + else: + example = dict([(k, v[0]) for k, v in batch.items()]) + return self._infer_feature_from_example(example) + + def _infer_feature_from_example(self, example): + if isinstance(self.features, Features): + return self.features + else: + for features in self.features: + try: + self._enforce_nested_string_type(features, example) + features.encode_example(example) + return features + except (ValueError, TypeError): + continue + feature_strings = "\n".join([f"Feature option {i}: {feature}" for i, feature in enumerate(self.features)]) + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format:\n{feature_strings},\n" + f"Input predictions: {summarize_if_long_list(example['predictions'])},\n" + f"Input references: {summarize_if_long_list(example['references'])}" + ) + raise ValueError(error_msg) from None + + def _feature_names(self): + if isinstance(self.features, list): + feature_names = list(self.features[0].keys()) + else: + feature_names = list(self.features.keys()) + return feature_names + + def _init_writer(self, timeout=1): + if self.num_process > 1: + if self.process_id == 0: + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + self.rendez_vous_lock = FileLock(file_path) + try: + self.rendez_vous_lock.acquire(timeout=timeout) + except TimeoutError: + raise ValueError( + f"Error in _init_writer: another evalution module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + + if self.keep_in_memory: + self.buf_writer = pa.BufferOutputStream() + self.writer = ArrowWriter( + features=self.selected_feature_format, stream=self.buf_writer, writer_batch_size=self.writer_batch_size + ) + else: + self.buf_writer = None + + # Get cache file name and lock it + if self.cache_file_name is None or self.filelock is None: + cache_file_name, filelock = self._create_cache_file() # get ready + self.cache_file_name = cache_file_name + self.filelock = filelock + + self.writer = ArrowWriter( + features=self.selected_feature_format, + path=self.cache_file_name, + writer_batch_size=self.writer_batch_size, + ) + # Setup rendez-vous here if + if self.num_process > 1: + if self.process_id == 0: + self._check_all_processes_locks() # wait for everyone to be ready + self.rendez_vous_lock.release() # let everyone go + else: + self._check_rendez_vous() # wait for master to be ready and to let everyone go + + def _info(self) -> EvaluationModuleInfo: + """Construct the EvaluationModuleInfo object. See `EvaluationModuleInfo` for details. + + Warning: This function is only called once and the result is cached for all + following .info() calls. + + Returns: + info: (EvaluationModuleInfo) The EvaluationModule information + """ + raise NotImplementedError + + def download_and_prepare( + self, + download_config: Optional[DownloadConfig] = None, + dl_manager: Optional[DownloadManager] = None, + ): + """Downloads and prepares evaluation module for reading. + + Args: + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + dl_manager ([`DownloadManager`], *optional*): + Specific download manager to use. + + Example: + + ```py + >>> import evaluate + ``` + """ + if dl_manager is None: + if download_config is None: + download_config = DownloadConfig() + download_config.cache_dir = os.path.join(self.data_dir, "downloads") + download_config.force_download = False + + dl_manager = DownloadManager( + dataset_name=self.name, download_config=download_config, data_dir=self.data_dir + ) + + self._download_and_prepare(dl_manager) + + def _download_and_prepare(self, dl_manager): + """Downloads and prepares resources for the evaluation module. + + This is the internal implementation to overwrite called when user calls + `download_and_prepare`. It should download all required resources for the evaluation module. + + Args: + dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. + """ + return None + + def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]: + """This method defines the common API for all the evaluation module in the library""" + raise NotImplementedError + + def __del__(self): + if hasattr(self, "filelock") and self.filelock is not None: + self.filelock.release() + if hasattr(self, "rendez_vous_lock") and self.rendez_vous_lock is not None: + self.rendez_vous_lock.release() + if hasattr(self, "writer"): # in case it was already deleted + del self.writer + if hasattr(self, "data"): # in case it was already deleted + del self.data + + def _enforce_nested_string_type(self, schema, obj): + """ + Recursively checks if there is any Value feature of type string and throws TypeError if corresponding object is not a string. + Since any Python object can be cast to string this avoids implicitly casting wrong input types (e.g. lists) to string without error. + """ + # Nested structures: we allow dict, list, tuples, sequences + if isinstance(schema, dict): + return [self._enforce_nested_string_type(sub_schema, o) for k, (sub_schema, o) in zip_dict(schema, obj)] + + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + return [self._enforce_nested_string_type(sub_schema, o) for o in obj] + elif isinstance(schema, Sequence): + # We allow to reverse list of dict => dict of list for compatiblity with tfds + if isinstance(schema.feature, dict): + if isinstance(obj, (list, tuple)): + # obj is a list of dict + for k, dict_tuples in zip_dict(schema.feature, *obj): + for sub_obj in dict_tuples[1:]: + if _check_non_null_non_empty_recursive(sub_obj, dict_tuples[0]): + self._enforce_nested_string_type(dict_tuples[0], sub_obj) + break + return None + else: + # obj is a single dict + for k, (sub_schema, sub_objs) in zip_dict(schema.feature, obj): + for sub_obj in sub_objs: + if _check_non_null_non_empty_recursive(sub_obj, sub_schema): + self._enforce_nested_string_type(sub_schema, sub_obj) + break + return None + # schema.feature is not a dict + if isinstance(obj, str): # don't interpret a string as a list + raise ValueError(f"Got a string but expected a list instead: '{obj}'") + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, schema.feature): + break + if not isinstance(first_elmt, list): + return self._enforce_nested_string_type(schema.feature, first_elmt) + + elif isinstance(schema, Value): + if pa.types.is_string(schema.pa_type) and not isinstance(obj, str): + raise TypeError(f"Expected type str but got {type(obj)}.") + + +class Metric(EvaluationModule): + """A Metric is the base class and common API for all metrics. + + Args: + config_name (`str`): + This is used to define a hash specific to a metric computation script and prevents the metric's data + to be overridden when the metric loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Metric.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + max_concurrent_cache_files (`int`): + Max number of concurrent metric cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Comparison(EvaluationModule): + """A Comparison is the base class and common API for all comparisons. + + Args: + config_name (`str`): + This is used to define a hash specific to a comparison computation script and prevents the comparison's data + to be overridden when the comparison loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Comparison.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + max_concurrent_cache_files (`int`): + Max number of concurrent comparison cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Measurement(EvaluationModule): + """A Measurement is the base class and common API for all measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a measurement computation script and prevents the measurement's data + to be overridden when the measurement loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Measurement.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + max_concurrent_cache_files (`int`): + Max number of concurrent measurement cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class CombinedEvaluations: + def __init__(self, evaluation_modules, force_prefix=False): + from .loading import load # avoid circular imports + + self.evaluation_module_names = None + if isinstance(evaluation_modules, list): + self.evaluation_modules = evaluation_modules + elif isinstance(evaluation_modules, dict): + self.evaluation_modules = list(evaluation_modules.values()) + self.evaluation_module_names = list(evaluation_modules.keys()) + loaded_modules = [] + + for module in self.evaluation_modules: + if isinstance(module, str): + module = load(module) + loaded_modules.append(module) + self.evaluation_modules = loaded_modules + + if self.evaluation_module_names is None: + self.evaluation_module_names = [module.name for module in self.evaluation_modules] + + self.force_prefix = force_prefix + + def add(self, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): + ... clf_metrics.add(references=ref, predictions=pred) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": prediction, "references": reference, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add(**batch) + + def add_batch(self, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... clf_metrics.add(references=refs, predictions=preds) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add_batch(**batch) + + def compute(self, predictions=None, references=None, **kwargs): + """Compute each evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (*optional*): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> clf_metrics.compute(predictions=[0,1], references=[1,1]) + {'accuracy': 0.5, 'f1': 0.6666666666666666} + ``` + """ + results = [] + + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + results.append(evaluation_module.compute(**batch)) + + return self._merge_results(results) + + def _merge_results(self, results): + merged_results = {} + results_keys = list(itertools.chain.from_iterable([r.keys() for r in results])) + duplicate_keys = {item for item, count in collections.Counter(results_keys).items() if count > 1} + + duplicate_names = [ + item for item, count in collections.Counter(self.evaluation_module_names).items() if count > 1 + ] + duplicate_counter = {name: 0 for name in duplicate_names} + + for module_name, result in zip(self.evaluation_module_names, results): + for k, v in result.items(): + if k not in duplicate_keys and not self.force_prefix: + merged_results[f"{k}"] = v + elif module_name in duplicate_counter: + merged_results[f"{module_name}_{duplicate_counter[module_name]}_{k}"] = v + else: + merged_results[f"{module_name}_{k}"] = v + + if module_name in duplicate_counter: + duplicate_counter[module_name] += 1 + + return merged_results + + +def combine(evaluations, force_prefix=False): + """Combines several metrics, comparisons, or measurements into a single `CombinedEvaluations` object that + can be used like a single evaluation module. + + If two scores have the same name, then they are prefixed with their module names. + And if two modules have the same name, please use a dictionary to give them different names, otherwise an integer id is appended to the prefix. + + Args: + evaluations (`Union[list, dict]`): + A list or dictionary of evaluation modules. The modules can either be passed + as strings or loaded `EvaluationModule`s. If a dictionary is passed its keys are the names used and the values the modules. + The names are used as prefix in case there are name overlaps in the returned results of each module or if `force_prefix=True`. + force_prefix (`bool`, *optional*, defaults to `False`): + If `True` all scores from the modules are prefixed with their name. If + a dictionary is passed the keys are used as name otherwise the module's name. + + Examples: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + ``` + """ + + return CombinedEvaluations(evaluations, force_prefix=force_prefix) diff --git a/ST/evaluate/src/evaluate/naming.py b/ST/evaluate/src/evaluate/naming.py new file mode 100644 index 0000000000000000000000000000000000000000..6335cf1b0ff47f0f2a409d6641fd5c528a31e949 --- /dev/null +++ b/ST/evaluate/src/evaluate/naming.py @@ -0,0 +1,82 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Utilities for file names.""" + +import itertools +import os +import re + + +_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])") +_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])") + +_single_underscore_re = re.compile(r"(?>> import evaluate + >>> result = {"bleu": 0.7} + >>> params = {"model": "gpt-2"} + >>> evaluate.save("./results/", **result, **params) + ``` + """ + current_time = datetime.now() + + file_path = _setup_path(path_or_file, current_time) + + data["_timestamp"] = current_time.isoformat() + data["_git_commit_hash"] = _git_commit_hash() + data["_evaluate_version"] = __version__ + data["_python_version"] = sys.version + data["_interpreter_path"] = sys.executable + + with FileLock(str(file_path) + ".lock"): + with open(file_path, "w") as f: + json.dump(data, f) + + # cleanup lock file + try: + os.remove(str(file_path) + ".lock") + except FileNotFoundError: + pass + + return file_path + + +def _setup_path(path_or_file, current_time): + path_or_file = Path(path_or_file) + is_file = len(path_or_file.suffix) > 0 + if is_file: + folder = path_or_file.parent + file_name = path_or_file.name + else: + folder = path_or_file + file_name = "result-" + current_time.strftime("%Y_%m_%d-%H_%M_%S") + ".json" + folder.mkdir(parents=True, exist_ok=True) + return folder / file_name + + +def _git_commit_hash(): + res = subprocess.run("git rev-parse --is-inside-work-tree".split(), cwd="./", stdout=subprocess.PIPE) + if res.stdout.decode().strip() == "true": + res = subprocess.run("git rev-parse HEAD".split(), cwd=os.getcwd(), stdout=subprocess.PIPE) + return res.stdout.decode().strip() + else: + return None diff --git a/ST/evaluate/src/evaluate/utils/__init__.py b/ST/evaluate/src/evaluate/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fac692eeb80def645bce9901324902ac9e1899f5 --- /dev/null +++ b/ST/evaluate/src/evaluate/utils/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa +# Lint as: python3 +"""Util import.""" + +__all__ = [ + "disable_progress_bar", + "enable_progress_bar", + "is_progress_bar_enabled", + "infer_gradio_input_types", + "json_to_string_type", + "parse_readme", + "parse_gradio_data", + "parse_test_cases", + "launch_gradio_widget", +] + +from .gradio import ( + infer_gradio_input_types, + json_to_string_type, + launch_gradio_widget, + parse_gradio_data, + parse_readme, + parse_test_cases, +) +from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled diff --git a/ST/evaluate/src/evaluate/utils/file_utils.py b/ST/evaluate/src/evaluate/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c52d44a958e328e498ce55562767cc0a81e63980 --- /dev/null +++ b/ST/evaluate/src/evaluate/utils/file_utils.py @@ -0,0 +1,618 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" + +import copy +import io +import json +import os +import posixpath +import re +import shutil +import sys +import tempfile +import time +import urllib +from contextlib import closing, contextmanager +from functools import partial +from hashlib import sha256 +from pathlib import Path +from typing import List, Optional, Type, TypeVar, Union +from urllib.parse import urljoin, urlparse + +import requests +from datasets import DownloadConfig +from datasets.utils.extract import ExtractManager +from datasets.utils.filelock import FileLock + +from .. import __version__, config +from . import logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +INCOMPLETE_SUFFIX = ".incomplete" + +T = TypeVar("T", str, Path) + + +def init_hf_modules(hf_modules_cache: Optional[Union[Path, str]] = None) -> str: + """ + Add hf_modules_cache to the python path. + By default hf_modules_cache='~/.cache/huggingface/modules'. + It can also be set with the environment variable HF_MODULES_CACHE. + This is used to add modules such as `datasets_modules` + """ + hf_modules_cache = hf_modules_cache if hf_modules_cache is not None else config.HF_MODULES_CACHE + hf_modules_cache = str(hf_modules_cache) + if hf_modules_cache not in sys.path: + sys.path.append(hf_modules_cache) + + os.makedirs(hf_modules_cache, exist_ok=True) + if not os.path.exists(os.path.join(hf_modules_cache, "__init__.py")): + with open(os.path.join(hf_modules_cache, "__init__.py"), "w"): + pass + return hf_modules_cache + + +def is_remote_url(url_or_filename: str) -> bool: + parsed = urlparse(url_or_filename) + return parsed.scheme in ("http", "https", "s3", "gs", "hdfs", "ftp") + + +def is_local_path(url_or_filename: str) -> bool: + # On unix the scheme of a local path is empty (for both absolute and relative), + # while on windows the scheme is the drive name (ex: "c") for absolute paths. + # for details on the windows behavior, see https://bugs.python.org/issue42215 + return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_relative_path(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename) + + +def relative_to_absolute_path(path: T) -> T: + """Convert relative path to absolute path.""" + abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path)))) + return Path(abs_path_str) if isinstance(path, Path) else abs_path_str + + +def hf_bucket_url(identifier: str, filename: str, use_cdn=False, dataset=True) -> str: + if dataset: + endpoint = config.CLOUDFRONT_DATASETS_DISTRIB_PREFIX if use_cdn else config.S3_DATASETS_BUCKET_PREFIX + else: + endpoint = config.CLOUDFRONT_METRICS_DISTRIB_PREFIX if use_cdn else config.S3_METRICS_BUCKET_PREFIX + return "/".join((endpoint, identifier, filename)) + + +def head_hf_s3( + identifier: str, filename: str, use_cdn=False, dataset=True, max_retries=0 +) -> Union[requests.Response, Exception]: + return http_head( + hf_bucket_url(identifier=identifier, filename=filename, use_cdn=use_cdn, dataset=dataset), + max_retries=max_retries, + ) + + +def hf_hub_url(path: str, name: str, revision: Optional[str] = None) -> str: + revision = revision or config.HUB_DEFAULT_VERSION + return config.HUB_EVALUATE_URL.format(path=path, name=name, revision=revision) + + +def url_or_path_join(base_name: str, *pathnames: str) -> str: + if is_remote_url(base_name): + return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames)) + else: + return Path(base_name, *pathnames).as_posix() + + +def url_or_path_parent(url_or_path: str) -> str: + if is_remote_url(url_or_path): + return url_or_path[: url_or_path.rindex("/")] + else: + return os.path.dirname(url_or_path) + + +def hash_url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".py"): + filename += ".py" + + return filename + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_EVALUATE_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + proxies=download_config.proxies, + resume_download=download_config.resume_download, + user_agent=download_config.user_agent, + local_files_only=download_config.local_files_only, + use_etag=download_config.use_etag, + max_retries=download_config.max_retries, + use_auth_token=download_config.use_auth_token, + ignore_url_params=download_config.ignore_url_params, + download_desc=download_config.download_desc, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + + return output_path + + +def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str: + ua = f"datasets/{__version__}; python/{config.PY_VERSION}" + ua += f"; pyarrow/{config.PYARROW_VERSION}" + if config.TORCH_AVAILABLE: + ua += f"; torch/{config.TORCH_VERSION}" + if config.TF_AVAILABLE: + ua += f"; tensorflow/{config.TF_VERSION}" + if config.JAX_AVAILABLE: + ua += f"; jax/{config.JAX_VERSION}" + if isinstance(user_agent, dict): + ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}" + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def get_authentication_headers_for_url(url: str, use_auth_token: Optional[Union[str, bool]] = None) -> dict: + """Handle the HF authentication""" + headers = {} + if url.startswith(config.HF_ENDPOINT): + token = None + if isinstance(use_auth_token, str): + token = use_auth_token + elif bool(use_auth_token): + from huggingface_hub import hf_api + + token = hf_api.HfFolder.get_token() + if token: + headers["authorization"] = f"Bearer {token}" + return headers + + +class OfflineModeIsEnabled(ConnectionError): + pass + + +def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None): + """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_EVALUATE_OFFLINE is True.""" + if config.HF_EVALUATE_OFFLINE: + raise OfflineModeIsEnabled( + "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg) + ) + + +def _retry( + func, + func_args: Optional[tuple] = None, + func_kwargs: Optional[dict] = None, + exceptions: Type[requests.exceptions.RequestException] = requests.exceptions.RequestException, + status_codes: Optional[List[int]] = None, + max_retries: int = 0, + base_wait_time: float = 0.5, + max_wait_time: float = 2, +): + func_args = func_args or () + func_kwargs = func_kwargs or {} + retry = 0 + while True: + try: + return func(*func_args, **func_kwargs) + except exceptions as err: + if retry >= max_retries or (status_codes and err.response.status_code not in status_codes): + raise err + else: + sleep_time = min(max_wait_time, base_wait_time * 2**retry) # Exponential backoff + logger.info(f"{func} timed out, retrying in {sleep_time}s... [{retry/max_retries}]") + time.sleep(sleep_time) + retry += 1 + + +def _request_with_retry( + method: str, + url: str, + max_retries: int = 0, + base_wait_time: float = 0.5, + max_wait_time: float = 2, + timeout: float = 10.0, + **params, +) -> requests.Response: + """Wrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. + + Note that if the environment variable HF_EVALUATE_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. + + Args: + method (str): HTTP method, such as 'GET' or 'HEAD'. + url (str): The URL of the resource to fetch. + max_retries (int): Maximum number of retries, defaults to 0 (no retries). + base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between + retries then grows exponentially, capped by max_wait_time. + max_wait_time (float): Maximum amount of time between two retries, in seconds. + **params: Params to pass to :obj:`requests.request`. + """ + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + tries, success = 0, False + while not success: + tries += 1 + try: + response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) + success = True + except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err: + if tries > max_retries: + raise err + else: + logger.info(f"{method} request to {url} timed out, retrying... [{tries/max_retries}]") + sleep_time = min(max_wait_time, base_wait_time * 2 ** (tries - 1)) # Exponential backoff + time.sleep(sleep_time) + return response + + +def ftp_head(url, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + r.read(1) + except Exception: + return False + return True + + +def ftp_get(url, temp_file, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + logger.info(f"Getting through FTP {url} into {temp_file.name}") + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + shutil.copyfileobj(r, temp_file) + except urllib.error.URLError as e: + raise ConnectionError(e) from None + + +def http_get( + url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=100.0, max_retries=0, desc=None +): + headers = copy.deepcopy(headers) or {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + if resume_size > 0: + headers["Range"] = f"bytes={resume_size:d}-" + response = _request_with_retry( + method="GET", + url=url, + stream=True, + proxies=proxies, + headers=headers, + cookies=cookies, + max_retries=max_retries, + timeout=timeout, + ) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get("Content-Length") + total = resume_size + int(content_length) if content_length is not None else None + with logging.tqdm( + unit="B", + unit_scale=True, + total=total, + initial=resume_size, + desc=desc or "Downloading", + disable=not logging.is_progress_bar_enabled(), + ) as progress: + for chunk in response.iter_content(chunk_size=1024): + progress.update(len(chunk)) + temp_file.write(chunk) + + +def http_head( + url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=0 +) -> requests.Response: + headers = copy.deepcopy(headers) or {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + response = _request_with_retry( + method="HEAD", + url=url, + proxies=proxies, + headers=headers, + cookies=cookies, + allow_redirects=allow_redirects, + timeout=timeout, + max_retries=max_retries, + ) + return response + + +def request_etag(url: str, use_auth_token: Optional[Union[str, bool]] = None) -> Optional[str]: + headers = get_authentication_headers_for_url(url, use_auth_token=use_auth_token) + response = http_head(url, headers=headers, max_retries=3) + response.raise_for_status() + etag = response.headers.get("ETag") if response.ok else None + return etag + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + proxies=None, + etag_timeout=100, + resume_download=False, + user_agent=None, + local_files_only=False, + use_etag=True, + max_retries=0, + use_auth_token=None, + ignore_url_params=False, + download_desc=None, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if cache_dir is None: + cache_dir = config.HF_EVALUATE_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + if ignore_url_params: + # strip all query parameters and #fragments from the URL + cached_url = urljoin(url, urlparse(url).path) + else: + cached_url = url # additional parameters may be added to the given URL + + connected = False + response = None + cookies = None + etag = None + head_error = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(cached_url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, use_auth_token=use_auth_token) + if user_agent is not None: + headers["user-agent"] = user_agent + + # We don't have the file locally or we need an eTag + if not local_files_only: + if url.startswith("ftp://"): + connected = ftp_head(url) + try: + response = http_head( + url, + allow_redirects=True, + proxies=proxies, + timeout=etag_timeout, + max_retries=max_retries, + headers=headers, + ) + if response.status_code == 200: # ok + etag = response.headers.get("ETag") if use_etag else None + for k, v in response.cookies.items(): + # In some edge cases, we need to get a confirmation token + if k.startswith("download_warning") and "drive.google.com" in url: + url += "&confirm=" + v + cookies = response.cookies + connected = True + # Fix Google Drive URL to avoid Virus scan warning + if "drive.google.com" in url and "confirm=" not in url: + url += "&confirm=t" + # In some edge cases, head request returns 400 but the connection is actually ok + elif ( + (response.status_code == 400 and "firebasestorage.googleapis.com" in url) + or (response.status_code == 405 and "drive.google.com" in url) + or ( + response.status_code == 403 + and ( + re.match(r"^https?://github.com/.*?/.*?/releases/download/.*?/.*?$", url) + or re.match(r"^https://.*?s3.*?amazonaws.com/.*?$", response.url) + ) + ) + or (response.status_code == 403 and "ndownloader.figstatic.com" in url) + ): + connected = True + logger.info(f"Couldn't get ETag version for url {url}") + elif response.status_code == 401 and config.HF_ENDPOINT in url and use_auth_token is None: + raise ConnectionError( + f"Unauthorized for URL {url}. Please use the parameter ``use_auth_token=True`` after logging in with ``huggingface-cli login``" + ) + except (OSError, requests.exceptions.Timeout) as e: + # not connected + head_error = e + pass + + # connected == False = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # try to get the last downloaded one + if not connected: + if os.path.exists(cache_path) and not force_download: + return cache_path + if local_files_only: + raise FileNotFoundError( + f"Cannot find the requested files in the cached path at {cache_path} and outgoing traffic has been" + " disabled. To enable file online look-ups, set 'local_files_only' to False." + ) + elif response is not None and response.status_code == 404: + raise FileNotFoundError(f"Couldn't find file at {url}") + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + if head_error is not None: + raise ConnectionError(f"Couldn't reach {url} ({repr(head_error)})") + elif response is not None: + raise ConnectionError(f"Couldn't reach {url} (error {response.status_code})") + else: + raise ConnectionError(f"Couldn't reach {url}") + + # Try a second time + filename = hash_url_to_filename(cached_url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # From now on, connected is True. + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + + if resume_download: + incomplete_path = cache_path + ".incomplete" + + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path, "a+b") as f: + yield f + + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 + + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + + # GET file object + if url.startswith("ftp://"): + ftp_get(url, temp_file) + else: + http_get( + url, + temp_file, + proxies=proxies, + resume_size=resume_size, + headers=headers, + cookies=cookies, + max_retries=max_retries, + desc=download_desc, + ) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr) + return fn + + return docstring_decorator + + +def estimate_dataset_size(paths): + return sum(path.stat().st_size for path in paths) + + +def readline(f: io.RawIOBase): + # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525 + res = bytearray() + while True: + b = f.read(1) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) diff --git a/ST/evaluate/src/evaluate/utils/gradio.py b/ST/evaluate/src/evaluate/utils/gradio.py new file mode 100644 index 0000000000000000000000000000000000000000..3b73d9c67e711caad66edaf0f66808fff296aabd --- /dev/null +++ b/ST/evaluate/src/evaluate/utils/gradio.py @@ -0,0 +1,131 @@ +import json +import os +import re +import sys +from pathlib import Path + +import numpy as np +from datasets import Value + +from .logging import get_logger + + +logger = get_logger(__name__) + +REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]") + + +def infer_gradio_input_types(feature_types): + """ + Maps metric feature types to input types for gradio Dataframes: + - float/int -> numbers + - string -> strings + - any other -> json + Note that json is not a native gradio type but will be treated as string that + is then parsed as a json. + """ + input_types = [] + for feature_type in feature_types: + input_type = "json" + if isinstance(feature_type, Value): + if feature_type.dtype.startswith("int") or feature_type.dtype.startswith("float"): + input_type = "number" + elif feature_type.dtype == "string": + input_type = "str" + input_types.append(input_type) + return input_types + + +def json_to_string_type(input_types): + """Maps json input type to str.""" + return ["str" if i == "json" else i for i in input_types] + + +def parse_readme(filepath): + """Parses a repositories README and removes""" + if not os.path.exists(filepath): + return "No README.md found." + with open(filepath, "r") as f: + text = f.read() + match = REGEX_YAML_BLOCK.search(text) + if match: + text = text[match.end() :] + return text + + +def parse_gradio_data(data, input_types): + """Parses data from gradio Dataframe for use in metric.""" + metric_inputs = {} + data.replace("", np.nan, inplace=True) + data.dropna(inplace=True) + for feature_name, input_type in zip(data, input_types): + if input_type == "json": + metric_inputs[feature_name] = [json.loads(d) for d in data[feature_name].to_list()] + elif input_type == "str": + metric_inputs[feature_name] = [d.strip('"') for d in data[feature_name].to_list()] + else: + metric_inputs[feature_name] = data[feature_name] + return metric_inputs + + +def parse_test_cases(test_cases, feature_names, input_types): + """ + Parses test cases to be used in gradio Dataframe. Note that an apostrophe is added + to strings to follow the format in json. + """ + if len(test_cases) == 0: + return None + examples = [] + for test_case in test_cases: + parsed_cases = [] + for feat, input_type in zip(feature_names, input_types): + if input_type == "json": + parsed_cases.append([str(element) for element in test_case[feat]]) + elif input_type == "str": + parsed_cases.append(['"' + element + '"' for element in test_case[feat]]) + else: + parsed_cases.append(test_case[feat]) + examples.append([list(i) for i in zip(*parsed_cases)]) + return examples + + +def launch_gradio_widget(metric): + """Launches `metric` widget with Gradio.""" + + try: + import gradio as gr + except ImportError as error: + logger.error("To create a metric widget with Gradio make sure gradio is installed.") + raise error + + local_path = Path(sys.path[0]) + # if there are several input types, use first as default. + if isinstance(metric.features, list): + (feature_names, feature_types) = zip(*metric.features[0].items()) + else: + (feature_names, feature_types) = zip(*metric.features.items()) + gradio_input_types = infer_gradio_input_types(feature_types) + + def compute(data): + return metric.compute(**parse_gradio_data(data, gradio_input_types)) + + iface = gr.Interface( + fn=compute, + inputs=gr.inputs.Dataframe( + headers=feature_names, + col_count=len(feature_names), + row_count=1, + datatype=json_to_string_type(gradio_input_types), + ), + outputs=gr.outputs.Textbox(label=metric.name), + description=( + metric.info.description + "\nIf this is a text-based metric, make sure to wrap you input in double quotes." + " Alternatively you can use a JSON-formatted list as input." + ), + title=f"Metric: {metric.name}", + article=parse_readme(local_path / "README.md"), + # TODO: load test cases and use them to populate examples + # examples=[parse_test_cases(test_cases, feature_names, gradio_input_types)] + ) + + iface.launch() diff --git a/ST/evaluate/src/evaluate/utils/logging.py b/ST/evaluate/src/evaluate/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..8df58d3dcfb4c8b903b78c244387561dc659e423 --- /dev/null +++ b/ST/evaluate/src/evaluate/utils/logging.py @@ -0,0 +1,234 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Logging utilities. """ + +import logging +import os +from logging import CRITICAL # NOQA +from logging import DEBUG # NOQA +from logging import ERROR # NOQA +from logging import FATAL # NOQA +from logging import INFO # NOQA +from logging import NOTSET # NOQA +from logging import WARN # NOQA +from logging import WARNING # NOQA +from typing import Optional + +from tqdm import auto as tqdm_lib + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If EVALUATE_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("EVALUATE_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option EVALUATE_VERBOSITY={env_level_str}, " + f"has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger with the specified name.""" + if name is None: + name = _get_library_name() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the Hugging Face Evaluate library's root logger. + Returns: + Logging level, e.g., `evaluate.logging.DEBUG` and `evaluate.logging.INFO`. + + + + Hugging Face Evaluate library has following logging levels: + - `evaluate.logging.CRITICAL`, `evaluate.logging.FATAL` + - `evaluate.logging.ERROR` + - `evaluate.logging.WARNING`, `evaluate.logging.WARN` + - `evaluate.logging.INFO` + - `evaluate.logging.DEBUG` + + + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """Set the level for the Hugging Face Evaluate library's root logger. + Args: + verbosity: + Logging level, e.g., `evaluate.logging.DEBUG` and `evaluate.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the level for the Hugging Face Evaluate library's root logger to `INFO`. + + This will display most of the logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.INFO)`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the level for the Hugging Face Evaluate library's root logger to `WARNING`. + + This will display only the warning and errors logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.WARNING)`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the level for the Hugging Face Evaluate library's root logger to `DEBUG`. + + This will display all the logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.DEBUG)`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the level for the Hugging Face Evaluate library's root logger to `ERROR`. + + This will display only the errors logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.ERROR)`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """Disable propagation of the library log outputs. + Note that log propagation is disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """Enable propagation of the library log outputs. + Please disable the Hugging Face Evaluate library's default handler to prevent double logging if the root logger has + been configured. + """ + _get_library_root_logger().propagate = True + + +# Configure the library root logger at the module level (singleton-like) +_configure_library_root_logger() + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +_tqdm_active = True + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + + +def disable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False diff --git a/ST/evaluate/src/evaluate/visualization.py b/ST/evaluate/src/evaluate/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..b8be8605805e4d11ef93a2911ab32afe934a78bb --- /dev/null +++ b/ST/evaluate/src/evaluate/visualization.py @@ -0,0 +1,230 @@ +import textwrap + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +class ComplexRadar: + """Create a complex radar chart with different scales for each variable + Args: + fig (`matplotlib.figure`) : A matplotlib figure object to add the axes on. + variables (`list`) : a list of variables to. plot + ranges (`list` of `tuples`): A list of ranges (min, max) for each variable + n_ring_levels (`int): Number of ordinate or ring levels to draw. + Default: 5. + show_scales (`bool`): Indicates if we the ranges for each variable are plotted. + Default: True. + format_cfg (`dict`): A dictionary with formatting configurations. + Default: None. + Returns: + `matplotlib.figure.Figure`: a radar plot. + """ + + def __init__(self, fig, variables, ranges, n_ring_levels=5, show_scales=True, format_cfg=None): + + self.format_cfg = format_cfg + + # Calculate angles and create for each variable an axes + # Consider here the trick with having the first axes element twice (len+1) + angles = np.arange(0, 360, 360.0 / len(variables)) + axes = [ + fig.add_axes([0.1, 0.1, 0.9, 0.9], polar=True, label="axes{}".format(i), **self.format_cfg["axes_args"]) + for i in range(len(variables) + 1) + ] + + # Ensure clockwise rotation (first variable at the top N) + for ax in axes: + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + ax.set_axisbelow(True) + + # Writing the ranges on each axes + for i, ax in enumerate(axes): + + # Here we do the trick by repeating the first iteration + j = 0 if (i == 0 or i == 1) else i - 1 + ax.set_ylim(*ranges[j]) + # Set endpoint to True if you like to have values right before the last circle + grid = np.linspace(*ranges[j], num=n_ring_levels, endpoint=self.format_cfg["incl_endpoint"]) + gridlabel = ["{}".format(round(x, 2)) for x in grid] + gridlabel[0] = "" # remove values from the center + lines, labels = ax.set_rgrids( + grid, labels=gridlabel, angle=angles[j], **self.format_cfg["rgrid_tick_lbls_args"] + ) + + ax.set_ylim(*ranges[j]) + ax.spines["polar"].set_visible(False) + ax.grid(visible=False) + + if show_scales is False: + ax.set_yticklabels([]) + + # Set all axes except the first one unvisible + for ax in axes[1:]: + ax.patch.set_visible(False) + ax.xaxis.set_visible(False) + + # Setting the attributes + self.angle = np.deg2rad(np.r_[angles, angles[0]]) + self.ranges = ranges + self.ax = axes[0] + self.ax1 = axes[1] + self.plot_counter = 0 + + # Draw (inner) circles and lines + self.ax.yaxis.grid(**self.format_cfg["rad_ln_args"]) + # Draw outer circle + self.ax.spines["polar"].set(**self.format_cfg["outer_ring"]) + # Draw angle lines + self.ax.xaxis.grid(**self.format_cfg["angle_ln_args"]) + + # ax1 is the duplicate of axes[0] (self.ax) + # Remove everything from ax1 except the plot itself + self.ax1.axis("off") + self.ax1.set_zorder(9) + + # Create the outer labels for each variable + l, text = self.ax.set_thetagrids(angles, labels=variables) + + # Beautify them + labels = [t.get_text() for t in self.ax.get_xticklabels()] + labels = [ + "\n".join( + textwrap.wrap( + label, + self.format_cfg["theta_tick_lbls_txt_wrap"], + break_long_words=self.format_cfg["theta_tick_lbls_brk_lng_wrds"], + ) + ) + for label in labels + ] + self.ax.set_xticklabels(labels, **self.format_cfg["theta_tick_lbls"]) + + for t, a in zip(self.ax.get_xticklabels(), angles): + if a == 0: + t.set_ha("center") + elif a > 0 and a < 180: + t.set_ha("left") + elif a == 180: + t.set_ha("center") + else: + t.set_ha("right") + + self.ax.tick_params(axis="both", pad=self.format_cfg["theta_tick_lbls_pad"]) + + def _scale_data(self, data, ranges): + """Scales data[1:] to ranges[0]""" + for d, (y1, y2) in zip(data[1:], ranges[1:]): + assert (y1 <= d <= y2) or (y2 <= d <= y1) + x1, x2 = ranges[0] + d = data[0] + sdata = [d] + for d, (y1, y2) in zip(data[1:], ranges[1:]): + sdata.append((d - y1) / (y2 - y1) * (x2 - x1) + x1) + return sdata + + def plot(self, data, *args, **kwargs): + """Plots a line""" + sdata = self._scale_data(data, self.ranges) + self.ax1.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kwargs) + self.plot_counter = self.plot_counter + 1 + + def use_legend(self, *args, **kwargs): + """Shows a legend""" + self.ax1.legend(*args, **kwargs) + + +def radar_plot(data, model_names, invert_range=[], config=None, fig=None): + """Create a complex radar chart with different scales for each variable + Source: https://towardsdatascience.com/how-to-create-and-visualize-complex-radar-charts-f7764d0f3652 + + Args: + data (`List[dict]`): the results (list of metric + value pairs). + E.g. data = [{"accuracy": 0.9, "precision":0.8},{"accuracy": 0.7, "precision":0.6}] + names (`List[dict]`): model names. + E.g. names = ["model1", "model 2", ...] + invert_range (`List[dict]`, optional): the metrics to invert (in cases when smaller is better, e.g. speed) + E.g. invert_range=["latency_in_seconds"] + config (`dict`, optional) : a specification of the formatting configurations, namely: + + - rad_ln_args (`dict`, default `{"visible": True}`): The visibility of the radial (circle) lines. + + - outer_ring (`dict`, default `{"visible": True}`): The visibility of the outer ring. + + - angle_ln_args (`dict`, default `{"visible": True}`): The visibility of the angle lines. + + - rgrid_tick_lbls_args (`dict`, default `{"fontsize": 12}`): The font size of the tick labels on the scales. + + - theta_tick_lbls (`dict`, default `{"fontsize": 12}`): The font size of the variable labels on the plot. + + - theta_tick_lbls_pad (`int`, default `3`): The padding of the variable labels on the plot. + + - theta_tick_lbls_brk_lng_wrds (`bool`, default `True` ): Whether long words in the label are broken up or not. + + - theta_tick_lbls_txt_wrap (`int`, default `15`): Text wrap for tick labels + + - incl_endpoint (`bool`, default `False`): Include value endpoints on calse + + - marker (`str`, default `"o"`): the shape of the marker used in the radar plot. + + - markersize (`int`, default `3`): the shape of the marker used in the radar plot. + + - legend_loc (`str`, default `"upper right"`): the location of the legend in the radar plot. Must be one of: 'upper left', 'upper right', 'lower left', 'lower right'. + + - bbox_to_anchor (`tuple`, default `(2, 1)`: anchor for the legend. + fig (`matplotlib.figure.Figure`, optional): figure used to plot the radar plot. + + Returns: + `matplotlib.figure.Figure` + """ + data = pd.DataFrame(data) + data.index = model_names + variables = data.keys() + if all(x in variables for x in invert_range) is False: + raise ValueError("All of the metrics in `invert_range` should be in the data provided.") + min_max_per_variable = data.describe().T[["min", "max"]] + min_max_per_variable["min"] = min_max_per_variable["min"] - 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + min_max_per_variable["max"] = min_max_per_variable["max"] + 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + + ranges = list(min_max_per_variable.itertuples(index=False, name=None)) + ranges = [ + (max_value, min_value) if var in invert_range else (min_value, max_value) + for var, (min_value, max_value) in zip(variables, ranges) + ] + format_cfg = { + "axes_args": {}, + "rad_ln_args": {"visible": True}, + "outer_ring": {"visible": True}, + "angle_ln_args": {"visible": True}, + "rgrid_tick_lbls_args": {"fontsize": 12}, + "theta_tick_lbls": {"fontsize": 12}, + "theta_tick_lbls_pad": 3, + "theta_tick_lbls_brk_lng_wrds": True, + "theta_tick_lbls_txt_wrap": 15, + "incl_endpoint": False, + "marker": "o", + "markersize": 3, + "legend_loc": "upper right", + "bbox_to_anchor": (2, 1), + } + if config is not None: + format_cfg.update(config) + if fig is None: + fig = plt.figure() + radar = ComplexRadar( + fig, + variables, + ranges, + n_ring_levels=3, + show_scales=True, + format_cfg=format_cfg, + ) + for g in zip(data.index): + radar.plot(data.loc[g].values, label=g, marker=format_cfg["marker"], markersize=format_cfg["markersize"]) + radar.use_legend(**{"loc": format_cfg["legend_loc"], "bbox_to_anchor": format_cfg["bbox_to_anchor"]}) + return fig diff --git a/ST/evaluate/templates/cookiecutter.json b/ST/evaluate/templates/cookiecutter.json new file mode 100644 index 0000000000000000000000000000000000000000..44ad5052d08a72f946798ef6e572059541bb73ec --- /dev/null +++ b/ST/evaluate/templates/cookiecutter.json @@ -0,0 +1,9 @@ +{ + "module_name": "Awesome Module", + "module_type": "module", + "module_description": "This new module is designed to solve this great ML task and is crafted with a lot of care and love.", + "module_slug": "{{ cookiecutter.module_name|lower|replace(' ', '_') }}", + "module_class_name": "{{ cookiecutter.module_name|replace(' ', '') }}", + "namespace": "", + "dataset_name": "" +} \ No newline at end of file diff --git a/ST/evaluate/templates/{{ cookiecutter.module_slug }}/README.md b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1168061ad8adf4105e06a21741c615ec93acea99 --- /dev/null +++ b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/README.md @@ -0,0 +1,50 @@ +--- +title: {{ cookiecutter.module_name }} +datasets: +- {{ cookiecutter.dataset_name }} +tags: +- evaluate +- {{ cookiecutter.module_type }} +description: "TODO: add a description here" +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +--- + +# {{ cookiecutter.module_type|capitalize }} Card for {{ cookiecutter.module_name }} + +***Module Card Instructions:*** *Fill out the following subsections. Feel free to take a look at existing {{ cookiecutter.module_type }} cards if you'd like examples.* + +## {{ cookiecutter.module_type|capitalize }} Description +*Give a brief overview of this {{ cookiecutter.module_type }}, including what task(s) it is usually used for, if any.* + +## How to Use +*Give general statement of how to use the {{ cookiecutter.module_type }}* + +*Provide simplest possible example for using the {{ cookiecutter.module_type }}* + +### Inputs +*List all input arguments in the format below* +- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).* + +### Output Values + +*Explain what this {{ cookiecutter.module_type }} outputs and provide an example of what the {{ cookiecutter.module_type }} output looks like. Modules should return a dictionary with one or multiple key-value pairs, e.g. {"bleu" : 6.02}* + +*State the range of possible values that the {{ cookiecutter.module_type }}'s output can take, as well as what in that range is considered good. For example: "This {{ cookiecutter.module_type }} can take on any value between 0 and 100, inclusive. Higher scores are better."* + +#### Values from Popular Papers +*Give examples, preferrably with links to leaderboards or publications, to papers that have reported this {{ cookiecutter.module_type }}, along with the values they have reported.* + +### Examples +*Give code examples of the {{ cookiecutter.module_type }} being used. Try to include examples that clear up any potential ambiguity left from the {{ cookiecutter.module_type }} description above. If possible, provide a range of examples that show both typical and atypical results, as well as examples where a variety of input parameters are passed.* + +## Limitations and Bias +*Note any known limitations or biases that the {{ cookiecutter.module_type }} has, with links and references if possible.* + +## Citation +*Cite the source where this {{ cookiecutter.module_type }} was introduced.* + +## Further References +*Add any useful further references.* diff --git a/ST/evaluate/templates/{{ cookiecutter.module_slug }}/app.py b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6582ec47a21ff99a2b3e5696820f3b029b99a486 --- /dev/null +++ b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("{{ cookiecutter.namespace }}/{{ cookiecutter.module_slug }}") +launch_gradio_widget(module) \ No newline at end of file diff --git a/ST/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..630637afffa554a9ac5588a774f4e0bf8b81a2fe --- /dev/null +++ b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@main \ No newline at end of file diff --git a/ST/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..601ed757507caebec67493462d11eb4c8901c2a1 --- /dev/null +++ b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py @@ -0,0 +1,17 @@ +test_cases = [ + { + "predictions": [0, 0], + "references": [1, 1], + "result": {"metric_score": 0} + }, + { + "predictions": [1, 1], + "references": [1, 1], + "result": {"metric_score": 1} + }, + { + "predictions": [1, 0], + "references": [1, 1], + "result": {"metric_score": 0.5} + } +] \ No newline at end of file diff --git a/ST/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py new file mode 100644 index 0000000000000000000000000000000000000000..578ebd2e846fb3519c8cd3efb284b4c373eaa4ce --- /dev/null +++ b/ST/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py @@ -0,0 +1,95 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TODO: Add a description here.""" + +import evaluate +import datasets + + +# TODO: Add BibTeX citation +_CITATION = """\ +@InProceedings{huggingface:module, +title = {A great new module}, +authors={huggingface, Inc.}, +year={2020} +} +""" + +# TODO: Add description of the module here +_DESCRIPTION = """\ +This new module is designed to solve this great ML task and is crafted with a lot of care. +""" + + +# TODO: Add description of the arguments of the module here +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores +Args: + predictions: list of predictions to score. Each predictions + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. +Returns: + accuracy: description of the first score, + another_score: description of the second score, +Examples: + Examples should be written in doctest format, and should illustrate how + to use the function. + + >>> my_new_module = evaluate.load("my_new_module") + >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1]) + >>> print(results) + {'accuracy': 1.0} +""" + +# TODO: Define external resources urls if needed +BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class {{ cookiecutter.module_class_name }}(evaluate.{{ cookiecutter.module_type | capitalize}}): + """TODO: Short description of my evaluation module.""" + + def _info(self): + # TODO: Specifies the evaluate.EvaluationModuleInfo object + return evaluate.{{ cookiecutter.module_type | capitalize}}Info( + # This is the description that will appear on the modules page. + module_type="{{ cookiecutter.module_type}}", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features({ + 'predictions': datasets.Value('int64'), + 'references': datasets.Value('int64'), + }), + # Homepage of the module for documentation + homepage="http://module.homepage", + # Additional links to the codebase or references + codebase_urls=["http://github.com/path/to/codebase/of/new_module"], + reference_urls=["http://path.to.reference.url/new_module"] + ) + + def _download_and_prepare(self, dl_manager): + """Optional: download external resources useful to compute the scores""" + # TODO: Download external resources if needed + pass + + def _compute(self, predictions, references): + """Returns the scores""" + # TODO: Compute the different scores of the module + accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions) + return { + "accuracy": accuracy, + } \ No newline at end of file diff --git a/ST/evaluate/tests/__init__.py b/ST/evaluate/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ST/evaluate/tests/conftest.py b/ST/evaluate/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..1d0496df66452348c6490a37d2a7c9914f9d25e4 --- /dev/null +++ b/ST/evaluate/tests/conftest.py @@ -0,0 +1,466 @@ +import csv +import json +import lzma +import os +import tarfile +import textwrap + +import datasets +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from datasets import config +from datasets.arrow_dataset import Dataset +from datasets.features import ClassLabel, Features, Sequence, Value + + +@pytest.fixture(autouse=True) +def set_test_cache_config(tmp_path_factory, monkeypatch): + # test_hf_cache_home = tmp_path_factory.mktemp("cache") # TODO: why a cache dir per test function does not work? + test_hf_cache_home = tmp_path_factory.getbasetemp() / "cache" + test_hf_evaluate_cache = test_hf_cache_home / "datasets" + test_hf_metrics_cache = test_hf_cache_home / "metrics" + test_hf_modules_cache = test_hf_cache_home / "modules" + monkeypatch.setattr("evaluate.config.HF_EVALUATE_CACHE", str(test_hf_evaluate_cache)) + monkeypatch.setattr("evaluate.config.HF_METRICS_CACHE", str(test_hf_metrics_cache)) + monkeypatch.setattr("evaluate.config.HF_MODULES_CACHE", str(test_hf_modules_cache)) + test_DOWNLOADED_EVALUATE_PATH = test_hf_evaluate_cache / "downloads" + monkeypatch.setattr("evaluate.config.DOWNLOADED_EVALUATE_PATH", str(test_DOWNLOADED_EVALUATE_PATH)) + test_EXTRACTED_EVALUATE_PATH = test_hf_evaluate_cache / "downloads" / "extracted" + monkeypatch.setattr("evaluate.config.EXTRACTED_EVALUATE_PATH", str(test_EXTRACTED_EVALUATE_PATH)) + + +@pytest.fixture(autouse=True, scope="session") +def disable_tqdm_output(): + datasets.disable_progress_bar() + + +@pytest.fixture(autouse=True) +def set_update_download_counts_to_false(monkeypatch): + # don't take tests into account when counting downloads + monkeypatch.setattr("evaluate.config.HF_UPDATE_DOWNLOAD_COUNTS", False) + monkeypatch.setattr("datasets.config.HF_UPDATE_DOWNLOAD_COUNTS", False) + + +FILE_CONTENT = """\ + Text data. + Second line of data.""" + + +@pytest.fixture(scope="session") +def dataset(): + n = 10 + features = Features( + { + "tokens": Sequence(Value("string")), + "labels": Sequence(ClassLabel(names=["negative", "positive"])), + "answers": Sequence( + { + "text": Value("string"), + "answer_start": Value("int32"), + } + ), + "id": Value("int64"), + } + ) + dataset = Dataset.from_dict( + { + "tokens": [["foo"] * 5] * n, + "labels": [[1] * 5] * n, + "answers": [{"answer_start": [97], "text": ["1976"]}] * 10, + "id": list(range(n)), + }, + features=features, + ) + return dataset + + +@pytest.fixture(scope="session") +def arrow_file(tmp_path_factory, dataset): + filename = str(tmp_path_factory.mktemp("data") / "file.arrow") + dataset.map(cache_file_name=filename) + return filename + + +@pytest.fixture(scope="session") +def text_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.txt" + data = FILE_CONTENT + with open(filename, "w") as f: + f.write(data) + return filename + + +@pytest.fixture(scope="session") +def xz_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.txt.xz" + data = bytes(FILE_CONTENT, "utf-8") + with lzma.open(filename, "wb") as f: + f.write(data) + return filename + + +@pytest.fixture(scope="session") +def gz_file(tmp_path_factory): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "file.txt.gz") + data = bytes(FILE_CONTENT, "utf-8") + with gzip.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def bz2_file(tmp_path_factory): + import bz2 + + path = tmp_path_factory.mktemp("data") / "file.txt.bz2" + data = bytes(FILE_CONTENT, "utf-8") + with bz2.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def zstd_file(tmp_path_factory): + if config.ZSTANDARD_AVAILABLE: + import zstandard as zstd + + path = tmp_path_factory.mktemp("data") / "file.txt.zst" + data = bytes(FILE_CONTENT, "utf-8") + with zstd.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def lz4_file(tmp_path_factory): + if config.LZ4_AVAILABLE: + import lz4.frame + + path = tmp_path_factory.mktemp("data") / "file.txt.lz4" + data = bytes(FILE_CONTENT, "utf-8") + with lz4.frame.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def xml_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.xml" + data = textwrap.dedent( + """\ + + +
+ + + Contingut 1 + Content 1 + + + Contingut 2 + Content 2 + + + Contingut 3 + Content 3 + + + Contingut 4 + Content 4 + + + Contingut 5 + Content 5 + + + """ + ) + with open(filename, "w") as f: + f.write(data) + return filename + + +DATA = [ + {"col_1": "0", "col_2": 0, "col_3": 0.0}, + {"col_1": "1", "col_2": 1, "col_3": 1.0}, + {"col_1": "2", "col_2": 2, "col_3": 2.0}, + {"col_1": "3", "col_2": 3, "col_3": 3.0}, +] +DATA2 = [ + {"col_1": "4", "col_2": 4, "col_3": 4.0}, + {"col_1": "5", "col_2": 5, "col_3": 5.0}, +] +DATA_DICT_OF_LISTS = { + "col_1": ["0", "1", "2", "3"], + "col_2": [0, 1, 2, 3], + "col_3": [0.0, 1.0, 2.0, 3.0], +} + +DATA_312 = [ + {"col_3": 0.0, "col_1": "0", "col_2": 0}, + {"col_3": 1.0, "col_1": "1", "col_2": 1}, +] + +DATA_STR = [ + {"col_1": "s0", "col_2": 0, "col_3": 0.0}, + {"col_1": "s1", "col_2": 1, "col_3": 1.0}, + {"col_1": "s2", "col_2": 2, "col_3": 2.0}, + {"col_1": "s3", "col_2": 3, "col_3": 3.0}, +] + + +@pytest.fixture(scope="session") +def dataset_dict(): + return DATA_DICT_OF_LISTS + + +@pytest.fixture(scope="session") +def arrow_path(tmp_path_factory): + dataset = Dataset.from_dict(DATA_DICT_OF_LISTS) + path = str(tmp_path_factory.mktemp("data") / "dataset.arrow") + dataset.map(cache_file_name=path) + return path + + +@pytest.fixture(scope="session") +def csv_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.csv") + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["col_1", "col_2", "col_3"]) + writer.writeheader() + for item in DATA: + writer.writerow(item) + return path + + +@pytest.fixture(scope="session") +def csv2_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset2.csv") + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["col_1", "col_2", "col_3"]) + writer.writeheader() + for item in DATA: + writer.writerow(item) + return path + + +@pytest.fixture(scope="session") +def bz2_csv_path(csv_path, tmp_path_factory): + import bz2 + + path = tmp_path_factory.mktemp("data") / "dataset.csv.bz2" + with open(csv_path, "rb") as f: + data = f.read() + # data = bytes(FILE_CONTENT, "utf-8") + with bz2.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def zip_csv_path(csv_path, csv2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.csv.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(csv_path, arcname=os.path.basename(csv_path)) + f.write(csv2_path, arcname=os.path.basename(csv2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_csv_with_dir_path(csv_path, csv2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.csv.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(csv_path, arcname=os.path.join("main_dir", os.path.basename(csv_path))) + f.write(csv2_path, arcname=os.path.join("main_dir", os.path.basename(csv2_path))) + return path + + +@pytest.fixture(scope="session") +def parquet_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.parquet") + schema = pa.schema( + { + "col_1": pa.string(), + "col_2": pa.int64(), + "col_3": pa.float64(), + } + ) + with open(path, "wb") as f: + writer = pq.ParquetWriter(f, schema=schema) + pa_table = pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(DATA))] for k in DATA[0]}, schema=schema) + writer.write_table(pa_table) + writer.close() + return path + + +@pytest.fixture(scope="session") +def json_list_of_dicts_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.json") + data = {"data": DATA} + with open(path, "w") as f: + json.dump(data, f) + return path + + +@pytest.fixture(scope="session") +def json_dict_of_lists_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.json") + data = {"data": DATA_DICT_OF_LISTS} + with open(path, "w") as f: + json.dump(data, f) + return path + + +@pytest.fixture(scope="session") +def jsonl_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.jsonl") + with open(path, "w") as f: + for item in DATA: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl2_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset2.jsonl") + with open(path, "w") as f: + for item in DATA: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl_312_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset_312.jsonl") + with open(path, "w") as f: + for item in DATA_312: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl_str_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset-str.jsonl") + with open(path, "w") as f: + for item in DATA_STR: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def text_gz_path(tmp_path_factory, text_path): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "dataset.txt.gz") + with open(text_path, "rb") as orig_file: + with gzip.open(path, "wb") as zipped_file: + zipped_file.writelines(orig_file) + return path + + +@pytest.fixture(scope="session") +def jsonl_gz_path(tmp_path_factory, jsonl_path): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "dataset.jsonl.gz") + with open(jsonl_path, "rb") as orig_file: + with gzip.open(path, "wb") as zipped_file: + zipped_file.writelines(orig_file) + return path + + +@pytest.fixture(scope="session") +def zip_jsonl_path(jsonl_path, jsonl2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.jsonl.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(jsonl_path, arcname=os.path.basename(jsonl_path)) + f.write(jsonl2_path, arcname=os.path.basename(jsonl2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_jsonl_with_dir_path(jsonl_path, jsonl2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.jsonl.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(jsonl_path, arcname=os.path.join("main_dir", os.path.basename(jsonl_path))) + f.write(jsonl2_path, arcname=os.path.join("main_dir", os.path.basename(jsonl2_path))) + return path + + +@pytest.fixture(scope="session") +def tar_jsonl_path(jsonl_path, jsonl2_path, tmp_path_factory): + path = tmp_path_factory.mktemp("data") / "dataset.jsonl.tar" + with tarfile.TarFile(path, "w") as f: + f.add(jsonl_path, arcname=os.path.basename(jsonl_path)) + f.add(jsonl2_path, arcname=os.path.basename(jsonl2_path)) + return path + + +@pytest.fixture(scope="session") +def tar_nested_jsonl_path(tar_jsonl_path, jsonl_path, jsonl2_path, tmp_path_factory): + path = tmp_path_factory.mktemp("data") / "dataset_nested.jsonl.tar" + with tarfile.TarFile(path, "w") as f: + f.add(tar_jsonl_path, arcname=os.path.join("nested", os.path.basename(tar_jsonl_path))) + return path + + +@pytest.fixture(scope="session") +def text_path(tmp_path_factory): + data = ["0", "1", "2", "3"] + path = str(tmp_path_factory.mktemp("data") / "dataset.txt") + with open(path, "w") as f: + for item in data: + f.write(item + "\n") + return path + + +@pytest.fixture(scope="session") +def text2_path(tmp_path_factory): + data = ["0", "1", "2", "3"] + path = str(tmp_path_factory.mktemp("data") / "dataset2.txt") + with open(path, "w") as f: + for item in data: + f.write(item + "\n") + return path + + +@pytest.fixture(scope="session") +def zip_text_path(text_path, text2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.text.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(text_path, arcname=os.path.basename(text_path)) + f.write(text2_path, arcname=os.path.basename(text2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_text_with_dir_path(text_path, text2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.text.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(text_path, arcname=os.path.join("main_dir", os.path.basename(text_path))) + f.write(text2_path, arcname=os.path.join("main_dir", os.path.basename(text2_path))) + return path + + +@pytest.fixture(scope="session") +def text_path_with_unicode_new_lines(tmp_path_factory): + text = "\n".join(["First", "Second\u2029with Unicode new line", "Third"]) + path = str(tmp_path_factory.mktemp("data") / "dataset_with_unicode_new_lines.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(text) + return path diff --git a/ST/evaluate/tests/test_evaluation_suite.py b/ST/evaluate/tests/test_evaluation_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..ad77c9d71bb55b5e0bcaab4f90d2364adcb4ef62 --- /dev/null +++ b/ST/evaluate/tests/test_evaluation_suite.py @@ -0,0 +1,31 @@ +from unittest import TestCase + +from evaluate import EvaluationSuite +from tests.test_evaluator import DummyTextClassificationPipeline + + +class TestEvaluationSuite(TestCase): + def setUp(self): + # Check that the EvaluationSuite loads successfully + self.evaluation_suite = EvaluationSuite.load("evaluate/evaluation-suite-ci") + + # Setup a dummy model for usage with the EvaluationSuite + self.dummy_model = DummyTextClassificationPipeline() + + def test_running_evaluation_suite(self): + + # Check that the evaluation suite successfully runs + results = self.evaluation_suite.run(self.dummy_model) + + # Check that the results are correct + for r in results: + self.assertEqual(r["accuracy"], 0.5) + + # Check that correct number of tasks were run + self.assertEqual(len(results), 2) + + def test_empty_suite(self): + + self.empty_suite = self.evaluation_suite + self.empty_suite.suite = [] + self.assertRaises(ValueError, self.empty_suite.run, self.dummy_model) diff --git a/ST/evaluate/tests/test_evaluator.py b/ST/evaluate/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..b0e4d5a6f76caa0513e7a8ed3b28f487f12572ab --- /dev/null +++ b/ST/evaluate/tests/test_evaluator.py @@ -0,0 +1,1145 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 + +from time import sleep +from unittest import TestCase, mock + +import numpy as np +from datasets import ClassLabel, Dataset, Features, Sequence, Value +from PIL import Image +from transformers import ( + AutoConfig, + AutoFeatureExtractor, + AutoModelForAudioClassification, + AutoModelForImageClassification, + AutoModelForQuestionAnswering, + AutoModelForSequenceClassification, + AutoModelForTokenClassification, + AutoTokenizer, + pipeline, +) + +from evaluate import ( + AudioClassificationEvaluator, + AutomaticSpeechRecognitionEvaluator, + Evaluator, + ImageClassificationEvaluator, + QuestionAnsweringEvaluator, + Text2TextGenerationEvaluator, + TextClassificationEvaluator, + TextGenerationEvaluator, + TokenClassificationEvaluator, + evaluator, + load, +) + +from .utils import slow + + +class DummyTextGenerationPipeline: + def __init__(self, prefix="generated", task="text-generation", num_return_sequences=1): + self.task = task + self.prefix = prefix + self.num_return_sequences = num_return_sequences + + def __call__(self, inputs, **kwargs): + return [[{f"{self.prefix}_text": "Lorem ipsum"} for _ in range(self.num_return_sequences)] for _ in inputs] + + +class DummyText2TextGenerationPipeline: + def __init__(self, prefix="generated", task="text2text-generation"): + self.task = task + self.prefix = prefix + + def __call__(self, inputs, **kwargs): + return [{f"{self.prefix}_text": "Lorem ipsum"} for _ in inputs] + + +class DummyTextClassificationPipeline: + def __init__(self, sleep_time=None): + self.task = "text-classification" + self.sleep_time = sleep_time + + def __call__(self, inputs, **kwargs): + if self.sleep_time is not None: + sleep(self.sleep_time) + return [{"label": "NEGATIVE"} if i % 2 == 1 else {"label": "POSITIVE"} for i, _ in enumerate(inputs)] + + +class DummyImageClassificationPipeline: + def __init__(self): + self.task = "image-classification" + + def __call__(self, images, **kwargs): + return [[{"score": 0.9, "label": "yurt"}, {"score": 0.1, "label": "umbrella"}] for i, _ in enumerate(images)] + + +class DummyQuestionAnsweringPipeline: + def __init__(self, v2: bool): + self.task = "question-answering" + self.v2 = v2 + + def __call__(self, question, context, **kwargs): + if self.v2: + return [ + {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} + if i % 2 == 0 + else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + for i in range(len(question)) + ] + else: + return [{"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} for _ in question] + + +class DummyTokenClassificationPipeline: + def __init__(self): + self.task = "token-classification" + + def __call__(self, inputs, **kwargs): + result = [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 4, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + + return [result] + + +class DummyAutomaticSpeechRecognitionPipeline: + def __init__(self) -> None: + self.task = "automatic-speech-recognition" + + def __call__(self, inputs, **kwargs): + return [{"text": "Lorem ipsum"} for _ in inputs] + + +class DummyAudioClassificationPipeline: + def __init__(self): + self.task = "audio-classification" + + def __call__(self, audio, **kwargs): + return [[{"score": 0.9, "label": "yes"}, {"score": 0.1, "label": "no"}] for i, _ in enumerate(audio)] + + +class TestEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.default_ckpt = "hf-internal-testing/tiny-random-bert" + self.default_model = AutoModelForSequenceClassification.from_pretrained(self.default_ckpt, num_labels=2) + self.default_tokenizer = AutoTokenizer.from_pretrained(self.default_ckpt) + self.pipe = pipeline("text-classification", model=self.default_model, tokenizer=self.default_tokenizer) + self.evaluator = evaluator("text-classification") + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.label_mapping = {"LABEL_0": 0.0, "LABEL_1": 1.0} + + def test_wrong_task(self): + self.assertRaises(KeyError, evaluator, "bad_task") + + def test_device_placement(self): + orig_import = __import__ + + pt_mock = mock.Mock() + tf_mock = mock.Mock() + + # mock import of torch and tensorflow + def import_pt_tf_mock(name, *args): + if name == "torch": + if pt_available: + return pt_mock + else: + raise ImportError + if name == "tensorflow": + if tf_available: + return tf_mock + else: + raise ImportError + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_pt_tf_mock): + # neither pt or tf are available + pt_available = False + tf_available = False + self.assertEqual(Evaluator._infer_device(), -1) + + # pt available but no GPU + pt_available = True + pt_mock.cuda.is_available.return_value = False + self.assertEqual(Evaluator._infer_device(), -1) + + # pt available and GPU found + pt_mock.cuda.is_available.return_value = True + self.assertEqual(Evaluator._infer_device(), 0) + + # tf available but no GPU + pt_available = False + tf_available = True + tf_mock.config.list_physical_devices.return_value = [] + self.assertEqual(Evaluator._infer_device(), -1) + + # tf available and GPU found + tf_mock.config.list_physical_devices.return_value = ["GPU:0", "GPU:1"] + self.assertEqual(Evaluator._infer_device(), 0) + + # pt accelerator found and pipeline instantiated on CPU + pt_mock.cuda.is_available.return_value = True + self.assertRaises( + ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ) + + # tf accelerator found and pipeline instantiated on CPU + pt_available = False + tf_available = True + self.assertRaises( + ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ) + + def test_pipe_init(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + def test_model_init(self): + self.evaluator.compute( + model_or_pipeline=self.default_model, + tokenizer=self.default_tokenizer, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + def test_model_str_init(self): + self.evaluator.compute( + model_or_pipeline=self.default_ckpt, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + +class TestTextClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.default_model = "lvwerra/distilbert-imdb" + self.input_column = "text" + self.label_column = "label" + self.pipe = DummyTextClassificationPipeline() + self.perf_pipe = DummyTextClassificationPipeline(sleep_time=0.1) + self.evaluator = evaluator("text-classification") + self.label_mapping = {"NEGATIVE": 0.0, "POSITIVE": 1.0} + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + ) + + model = AutoModelForSequenceClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + self.assertEqual(results["accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + tokenizer=tokenizer, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_class_init(self): + evaluator = TextClassificationEvaluator() + self.assertEqual(evaluator.task, "text-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="f1", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["f1"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_data_loading(self): + + # Test passing in dataset by name with split + data = self.evaluator.load_data("evaluate/imdb-ci", split="test[:1]") + self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + + # Test passing in dataset by name without split and inferring the optimal split + data = self.evaluator.load_data("evaluate/imdb-ci") + self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + + # Test that it chooses the correct one (e.g. imdb only has train and test, but no validation) + self.assertEqual(data.split, "test") + + # Test that the data point returned is correct; this maps to the first example in the dataset + self.assertEqual(data[0]["text"], "I love movies about whales!") + + # Test loading subset of a dataset with the `name` field + data = self.evaluator.load_data("evaluate/glue-ci", subset="cola", split="test") + self.assertEqual(isinstance(data, Dataset), True) + + # Test loading subset of a dataset with the `name` field and having it infer the split + data = self.evaluator.load_data("evaluate/glue-ci", subset="cola") + self.assertEqual(isinstance(data, Dataset), True) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_bootstrap(self): + data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=data, + metric="accuracy", + label_mapping=self.label_mapping, + strategy="bootstrap", + n_resamples=10, + random_state=0, + ) + self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.33333, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498, 5) + + def test_perf(self): + results = self.evaluator.compute( + model_or_pipeline=self.perf_pipe, + data=self.data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + n_resamples=10, + random_state=0, + ) + self.assertEqual(results["accuracy"], 1.0) + self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) + self.assertAlmostEqual(results["samples_per_second"], len(self.data) / results["total_time_in_seconds"], 5) + self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(self.data), 5) + + def test_bootstrap_and_perf(self): + data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + + results = self.evaluator.compute( + model_or_pipeline=self.perf_pipe, + data=data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + strategy="bootstrap", + n_resamples=10, + random_state=0, + ) + self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.333333, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498285, 5) + self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) + self.assertAlmostEqual(results["samples_per_second"], len(data) / results["total_time_in_seconds"], 5) + self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(data), 5) + + +class TestTextClassificationEvaluatorTwoColumns(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "label": [1, 0], + "premise": ["great car", "great movie"], + "hypothesis": ["great vehicle", "horrible movie"], + } + ) + self.default_model = "prajjwal1/bert-tiny-mnli" + self.input_column = "premise" + self.second_input_column = "hypothesis" + self.label_column = "label" + self.pipe = DummyTextClassificationPipeline() + self.evaluator = evaluator("text-classification") + self.label_mapping = {"NEGATIVE": 0.0, "POSITIVE": 1.0} + self.label_mapping2 = {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2} + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column=self.input_column, + second_input_column=self.second_input_column, + label_column="label", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + second_input_column=self.second_input_column, + label_column=self.label_column, + label_mapping=self.label_mapping2, + ) + self.assertEqual(results["accuracy"], 1.0) + + model = AutoModelForSequenceClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + second_input_column=self.second_input_column, + tokenizer=tokenizer, + label_mapping=self.label_mapping2, + ) + self.assertEqual(results["accuracy"], 1.0) + + +class TestImageClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "label": [2, 2], + "image": [Image.new("RGB", (500, 500), (255, 255, 255)), Image.new("RGB", (500, 500), (170, 95, 170))], + } + ) + self.default_model = "lysandre/tiny-vit-random" + self.pipe = DummyImageClassificationPipeline() + self.evaluator = evaluator("image-classification") + self.label_mapping = AutoConfig.from_pretrained(self.default_model).label2id + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + model = AutoModelForImageClassification.from_pretrained(self.default_model) + feature_extractor = AutoFeatureExtractor.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + feature_extractor=feature_extractor, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_class_init(self): + evaluator = ImageClassificationEvaluator() + self.assertEqual(evaluator.task, "image-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + +class TestQuestionAnsweringEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], + "context": ["My name is Felix and I love cookies!", "Misa name is Felix and misa love cookies!"], + "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": ["Felix"], "answer_start": [13]}], + "question": ["What is my name?", "What is my name?"], + } + ) + self.data_v2 = Dataset.from_dict( + { + "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], + "context": ["My name is Felix and I love cookies!", "Let's explore the city!"], + "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": [], "answer_start": []}], + "question": ["What is my name?", "What is my name?"], + } + ) + + self.default_model = "mrm8488/bert-tiny-finetuned-squadv2" + self.pipe = DummyQuestionAnsweringPipeline(v2=False) + self.pipe_v2 = DummyQuestionAnsweringPipeline(v2=True) + self.evaluator = evaluator("question-answering") + + def test_pipe_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + @slow + def test_model_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 0) + self.assertEqual(results["f1"], 100 / 3) + + model = AutoModelForQuestionAnswering.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="squad", + tokenizer=tokenizer, + ) + self.assertEqual(results["exact_match"], 0) + self.assertEqual(results["f1"], 100 / 3) + + def test_class_init(self): + # squad_v1-like dataset + evaluator = QuestionAnsweringEvaluator() + self.assertEqual(evaluator.task, "question-answering") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + # squad_v2-like dataset + evaluator = QuestionAnsweringEvaluator() + self.assertEqual(evaluator.task, "question-answering") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe_v2, + data=self.data_v2, + metric="squad_v2", + ) + self.assertDictEqual( + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 100.0} + ) + + @slow + def test_default_pipe_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + data=self.data, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + # squad_v2-like dataset + results = self.evaluator.compute( + data=self.data_v2, + metric="squad_v2", + ) + self.assertDictEqual( + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 0.0} + ) + + def test_data_loading(self): + # Test passing in dataset by name with data_split + data = self.evaluator.load_data("evaluate/squad-ci", split="validation[:1]") + self.evaluator.prepare_data( + data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + ) + + # Test passing in dataset by name without data_split and inferring the optimal split + data = self.evaluator.load_data("evaluate/squad-ci") + self.evaluator.prepare_data( + data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + ) + + # Test that it chooses the correct one (e.g. squad only has train and validation, but no test) + self.assertEqual(data.split, "validation") + + # Test that the data point returned is correct; this maps to the first example in the squad-ci dataset + self.assertEqual(data[0]["id"], "56be4db0acb8001400a502ec") + + def test_overwrite_default_metric(self): + # squad_v1-like dataset + squad = load("squad") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=squad, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + +class TestTokenClassificationEvaluator(TestCase): + def setUp(self): + features = Features( + { + "tokens": Sequence(feature=Value(dtype="string")), + "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC"])), + } + ) + + self.data = Dataset.from_dict( + { + "tokens": [["New", "York", "a", "nice", "City", "."]], + "ner_tags": [[1, 2, 0, 0, 1, 0]], + }, + features=features, + ) + self.default_model = "hf-internal-testing/tiny-bert-for-token-classification" + self.pipe = DummyTokenClassificationPipeline() + self.evaluator = evaluator("token-classification") + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 0.5) + + model = AutoModelForTokenClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="seqeval", + tokenizer=tokenizer, + ) + self.assertEqual(results["overall_accuracy"], 0.5) + + def test_class_init(self): + evaluator = TokenClassificationEvaluator() + self.assertEqual(evaluator.task, "token-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + ) + self.assertEqual(results["overall_accuracy"], 2 / 3) + + def test_overwrite_default_metric(self): + accuracy = load("seqeval") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + ) + self.assertEqual(results["overall_accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 1.0) + + def test_data_loading(self): + # Test passing in dataset by name with data_split + data = self.evaluator.load_data("evaluate/conll2003-ci", split="validation[:1]") + self.evaluator.prepare_data( + data=data, + input_column="tokens", + label_column="ner_tags", + join_by=" ", + ) + + # Test passing in dataset by name without data_split and inferring the optimal split + data = self.evaluator.load_data("evaluate/conll2003-ci") + self.evaluator.prepare_data( + data=data, + input_column="tokens", + label_column="ner_tags", + join_by=" ", + ) + + # Test that it chooses the correct one (e.g. conll2003 has train, validation, test but should select test) + self.assertEqual(data.split, "test") + + # Test that the data point returned is correct; this maps to the first example in the dataset + self.assertEqual(data[0]["id"], "0") + + def test_wrong_task(self): + self.assertRaises(KeyError, evaluator, "bad_task") + + def test_words_to_offsets(self): + task_evaluator = evaluator("token-classification") + + words = ["This", "is", "a", "test", "."] + join_by = " " + + offsets = task_evaluator.words_to_offsets(words, join_by) + + self.assertListEqual([(0, 3), (5, 6), (8, 8), (10, 13), (15, 15)], offsets) + + words = ["日", "本", "語", "はなせるの?"] + join_by = "" + + offsets = task_evaluator.words_to_offsets(words, join_by) + + self.assertListEqual([(0, 0), (1, 1), (2, 2), (3, 8)], offsets) + + def test_predictions_processor(self): + task_evaluator = evaluator("token-classification") + join_by = " " + words = [["New", "York", "a", "nice", "City", "."]] + + # aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 4, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "I-LOC", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 6, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + +class TestTextGenerationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"text": ["Lorem ipsum"]}) + self.pipe = DummyTextGenerationPipeline(num_return_sequences=4) + self.evaluator = evaluator("text-generation") + + def test_class_init(self): + evaluator = TextGenerationEvaluator() + self.assertEqual(evaluator.task, "text-generation") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="word_count", + ) + self.assertIsInstance(results["unique_words"], int) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertIsInstance(results["unique_words"], int) + + def test_overwrite_default_metric(self): + word_length = load("word_length") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=word_length, + ) + self.assertIsInstance(results["average_word_length"], int) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="word_length", + ) + self.assertIsInstance(results["average_word_length"], int) + + def test_process_predictions_multiple_return_sequences(self): + processed_predictions = self.evaluator.predictions_processor( + [ + [{"generated_text": "A"}, {"generated_text": "B"}], + [{"generated_text": "C"}, {"generated_text": "D"}], + ] + ) + self.assertEqual(processed_predictions, {"data": ["A", "B", "C", "D"]}) + + +class TestText2TextGenerationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "text": ["Lorem ipsum"] * 4, + "label": ["Ipsum Lorem"] * 4, + } + ) + self.pipe = DummyText2TextGenerationPipeline() + self.evaluator = evaluator("text2text-generation") + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(results["bleu"], 0) + + def test_class_init(self): + evaluator = Text2TextGenerationEvaluator() + self.assertEqual(evaluator.task, "text2text-generation") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="bleu", + ) + self.assertEqual(results["bleu"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertEqual(results["bleu"], 0) + + def test_overwrite_default_metric(self): + rouge = load("rouge") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=rouge, + ) + self.assertEqual(results["rouge1"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="rouge", + ) + self.assertEqual(results["rouge1"], 1.0) + + def test_summarization(self): + pipe = DummyText2TextGenerationPipeline(task="summarization", prefix="summary") + e = evaluator("summarization") + + results = e.compute( + model_or_pipeline=pipe, + data=self.data, + ) + self.assertEqual(results["rouge1"], 1.0) + + def test_translation(self): + pipe = DummyText2TextGenerationPipeline(task="translation", prefix="translation") + e = evaluator("translation") + + results = e.compute( + model_or_pipeline=pipe, + data=self.data, + ) + self.assertEqual(results["bleu"], 0) + + +class TestAutomaticSpeechRecognitionEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "path": [ + # Examples copied from default speech model of + # `automic-speech-recognition` pipeline: + # https://huggingface.co/facebook/wav2vec2-base-960h + # https://github.com/huggingface/transformers/blob/main/src/transformers/pipelines/__init__.py#L161 + "https://cdn-media.huggingface.co/speech_samples/sample1.flac", + "https://cdn-media.huggingface.co/speech_samples/sample2.flac", + ], + "sentence": ["Ipsum Lorem"] * 2, + } + ) + self.pipe = DummyAutomaticSpeechRecognitionPipeline() + self.evaluator = evaluator("automatic-speech-recognition") + + def test_pipe_init(self): + print(self.evaluator) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + print(results) + self.assertEqual(results["wer"], 1.0) + + def test_class_init(self): + evaluator = AutomaticSpeechRecognitionEvaluator() + self.assertEqual(evaluator.task, "automatic-speech-recognition") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="wer", + ) + self.assertEqual(results["wer"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertGreater(results["wer"], 1.0) + + def test_overwrite_default_metric(self): + cer = load("cer") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=cer, + ) + self.assertEqual(results["cer"], 0.7272727272727273) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="cer", + ) + self.assertEqual(results["cer"], 0.7272727272727273) + + +class TestAudioClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + {"file": ["https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/1.flac"], "label": [11]} + ) + self.raw_data = Dataset.from_dict( + { + "audio": [ + np.array( + [-0.00048828, -0.00018311, -0.00137329, 0.00079346, 0.00091553, 0.00085449], dtype=np.float32 + ) + ], + "label": [11], + } + ) + self.default_model = "superb/wav2vec2-base-superb-ks" + self.pipe = DummyAudioClassificationPipeline() + self.evaluator = evaluator("audio-classification") + self.label_mapping = AutoConfig.from_pretrained(self.default_model).label2id + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_raw_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, data=self.raw_data, label_mapping=self.label_mapping, input_column="audio" + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + model = AutoModelForAudioClassification.from_pretrained(self.default_model) + feature_extractor = AutoFeatureExtractor.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + feature_extractor=feature_extractor, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_class_init(self): + evaluator = AudioClassificationEvaluator() + self.assertEqual(evaluator.task, "audio-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + results_raw = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.raw_data, + label_mapping=self.label_mapping, + metric="accuracy", + input_column="audio", + ) + self.assertEqual(results_raw["accuracy"], 0) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) diff --git a/ST/evaluate/tests/test_file_utils.py b/ST/evaluate/tests/test_file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1b1cdccd4d6498b9ea0db08177222def4f088abb --- /dev/null +++ b/ST/evaluate/tests/test_file_utils.py @@ -0,0 +1,56 @@ +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from evaluate.utils.file_utils import OfflineModeIsEnabled, cached_path, ftp_get, ftp_head, http_get, http_head + + +FILE_CONTENT = """\ + Text data. + Second line of data.""" + + +def test_cached_path_local(text_file): + # absolute path + text_file = str(Path(text_file).resolve()) + assert cached_path(text_file) == text_file + # relative path + text_file = str(Path(__file__).resolve().relative_to(Path(os.getcwd()))) + assert cached_path(text_file) == text_file + + +def test_cached_path_missing_local(tmp_path): + # absolute path + missing_file = str(tmp_path.resolve() / "__missing_file__.txt") + with pytest.raises(FileNotFoundError): + cached_path(missing_file) + # relative path + missing_file = "./__missing_file__.txt" + with pytest.raises(FileNotFoundError): + cached_path(missing_file) + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_cached_path_offline(): + with pytest.raises(OfflineModeIsEnabled): + cached_path("https://huggingface.co") + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_http_offline(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.html" + with pytest.raises(OfflineModeIsEnabled): + http_get("https://huggingface.co", temp_file=filename) + with pytest.raises(OfflineModeIsEnabled): + http_head("https://huggingface.co") + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_ftp_offline(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.html" + with pytest.raises(OfflineModeIsEnabled): + ftp_get("ftp://huggingface.co", temp_file=filename) + with pytest.raises(OfflineModeIsEnabled): + ftp_head("ftp://huggingface.co") diff --git a/ST/evaluate/tests/test_hub.py b/ST/evaluate/tests/test_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..055996b1092d1c7d064d448c8f1ebb84781928b9 --- /dev/null +++ b/ST/evaluate/tests/test_hub.py @@ -0,0 +1,187 @@ +import glob +from unittest import TestCase +from unittest.mock import patch + +import pytest +import requests +import yaml + +from evaluate.hub import push_to_hub +from tests.test_metric import DummyMetric + + +minimum_metadata = { + "model-index": [ + { + "results": [ + { + "task": {"type": "dummy-task"}, + "dataset": {"type": "dataset_type", "name": "dataset_name"}, + "metrics": [ + {"type": "dummy_metric", "value": 1.0, "name": "Pretty Metric Name"}, + ], + } + ] + } + ] +} + +extras_metadata = { + "model-index": [ + { + "results": [ + { + "task": {"type": "dummy-task", "name": "task_name"}, + "dataset": { + "type": "dataset_type", + "name": "dataset_name", + "config": "fr", + "split": "test", + "revision": "abc", + "args": {"a": 1, "b": 2}, + }, + "metrics": [ + { + "type": "dummy_metric", + "value": 1.0, + "name": "Pretty Metric Name", + "config": "default", + "args": {"hello": 1, "world": 2}, + }, + ], + } + ] + } + ] +} + + +@patch("evaluate.hub.HF_HUB_ALLOWED_TASKS", ["dummy-task"]) +@patch("evaluate.hub.dataset_info", lambda x: True) +@patch("evaluate.hub.model_info", lambda x: True) +@patch("evaluate.hub.metadata_update") +class TestHub(TestCase): + @pytest.fixture(autouse=True) + def inject_fixtures(self, caplog): + self._caplog = caplog + + def setUp(self): + self.metric = DummyMetric() + self.metric.add() + self.args = {"hello": 1, "world": 2} + self.result = self.metric.compute() + + def test_push_metric_required_arguments(self, metadata_update): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + metadata_update.assert_called_once_with(repo_id="username/repo", metadata=minimum_metadata, overwrite=False) + + def test_push_metric_missing_arguments(self, metadata_update): + with pytest.raises(TypeError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dummy-task", + ) + + def test_push_metric_invalid_arguments(self, metadata_update): + with pytest.raises(TypeError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + random_value="incorrect", + ) + + def test_push_metric_extra_arguments(self, metadata_update): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + dataset_config="fr", + dataset_split="test", + dataset_revision="abc", + dataset_args={"a": 1, "b": 2}, + task_type="dummy-task", + task_name="task_name", + metric_config=self.metric.config_name, + metric_args=self.args, + ) + + metadata_update.assert_called_once_with(repo_id="username/repo", metadata=extras_metadata, overwrite=False) + + def test_push_metric_invalid_task_type(self, metadata_update): + with pytest.raises(ValueError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="audio-classification", + ) + + def test_push_metric_invalid_dataset_type(self, metadata_update): + with patch("evaluate.hub.dataset_info") as mock_dataset_info: + mock_dataset_info.side_effect = requests.HTTPError() + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + assert "Dataset dataset_type not found on the Hub at hf.co/datasets/dataset_type" in self._caplog.text + metadata_update.assert_called_once_with( + repo_id="username/repo", metadata=minimum_metadata, overwrite=False + ) + + def test_push_metric_invalid_model_id(self, metadata_update): + with patch("evaluate.hub.model_info") as mock_model_info: + mock_model_info.side_effect = requests.HTTPError() + with pytest.raises(ValueError): + push_to_hub( + model_id="username/bad-repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + +class ValidateYaml(TestCase): + def setUp(self): + pass + + def testLoadingCards(self): + readme_filepaths = [] + for glob_path in ["measurements/*/README.md", "metrics/*/README.md", "comparisons/*/README.md"]: + readme_filepaths.extend(glob.glob(glob_path)) + for readme_file in readme_filepaths: + with open(readme_file, encoding="utf8") as f_yaml: + x = yaml.safe_load_all(f_yaml) + self.assertIsInstance(next(x), dict) diff --git a/ST/evaluate/tests/test_load.py b/ST/evaluate/tests/test_load.py new file mode 100644 index 0000000000000000000000000000000000000000..e20ea671fcb52874b498b5cb933818066296d9e2 --- /dev/null +++ b/ST/evaluate/tests/test_load.py @@ -0,0 +1,140 @@ +import importlib +import os +import tempfile +from unittest import TestCase + +import pytest +from datasets import DownloadConfig + +import evaluate +from evaluate.loading import ( + CachedEvaluationModuleFactory, + HubEvaluationModuleFactory, + LocalEvaluationModuleFactory, + evaluation_module_factory, +) + +from .utils import OfflineSimulationMode, offline + + +SAMPLE_METRIC_IDENTIFIER = "lvwerra/test" + +METRIC_LOADING_SCRIPT_NAME = "__dummy_metric1__" + +METRIC_LOADING_SCRIPT_CODE = """ +import evaluate +from evaluate import EvaluationModuleInfo +from datasets import Features, Value + +class __DummyMetric1__(evaluate.EvaluationModule): + + def _info(self): + return EvaluationModuleInfo(features=Features({"predictions": Value("int"), "references": Value("int")})) + + def _compute(self, predictions, references): + return {"__dummy_metric1__": sum(int(p == r) for p, r in zip(predictions, references))} +""" + + +@pytest.fixture +def metric_loading_script_dir(tmp_path): + script_name = METRIC_LOADING_SCRIPT_NAME + script_dir = tmp_path / script_name + script_dir.mkdir() + script_path = script_dir / f"{script_name}.py" + with open(script_path, "w") as f: + f.write(METRIC_LOADING_SCRIPT_CODE) + return str(script_dir) + + +class ModuleFactoryTest(TestCase): + @pytest.fixture(autouse=True) + def inject_fixtures(self, metric_loading_script_dir): + self._metric_loading_script_dir = metric_loading_script_dir + + def setUp(self): + self.hf_modules_cache = tempfile.mkdtemp() + self.cache_dir = tempfile.mkdtemp() + self.download_config = DownloadConfig(cache_dir=self.cache_dir) + self.dynamic_modules_path = evaluate.loading.init_dynamic_modules( + name="test_datasets_modules_" + os.path.basename(self.hf_modules_cache), + hf_modules_cache=self.hf_modules_cache, + ) + + def test_HubEvaluationModuleFactory_with_internal_import(self): + # "squad_v2" requires additional imports (internal) + factory = HubEvaluationModuleFactory( + "evaluate-metric/squad_v2", + module_type="metric", + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_HubEvaluationModuleFactory_with_external_import(self): + # "bleu" requires additional imports (external from github) + factory = HubEvaluationModuleFactory( + "evaluate-metric/bleu", + module_type="metric", + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_HubEvaluationModuleFactoryWithScript(self): + factory = HubEvaluationModuleFactory( + SAMPLE_METRIC_IDENTIFIER, + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_LocalMetricModuleFactory(self): + path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") + factory = LocalEvaluationModuleFactory( + path, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_CachedMetricModuleFactory(self): + path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") + factory = LocalEvaluationModuleFactory( + path, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + module_factory_result = factory.get_module() + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + factory = CachedEvaluationModuleFactory( + METRIC_LOADING_SCRIPT_NAME, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_cache_with_remote_canonical_module(self): + metric = "accuracy" + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + def test_cache_with_remote_community_module(self): + metric = "lvwerra/test" + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) diff --git a/ST/evaluate/tests/test_metric.py b/ST/evaluate/tests/test_metric.py new file mode 100644 index 0000000000000000000000000000000000000000..598b0f9295a903002b367a629bb3cffbfd47402e --- /dev/null +++ b/ST/evaluate/tests/test_metric.py @@ -0,0 +1,759 @@ +import os +import pickle +import tempfile +import time +from multiprocessing import Pool +from unittest import TestCase, mock + +import pytest +from datasets.features import Features, Sequence, Value + +from evaluate.module import EvaluationModule, EvaluationModuleInfo, combine + +from .utils import require_tf, require_torch + + +class DummyMetric(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features({"predictions": Value("int64"), "references": Value("int64")}), + ) + + def _compute(self, predictions, references): + result = {} + if not predictions: + return result + else: + result["accuracy"] = sum(i == j for i, j in zip(predictions, references)) / len(predictions) + try: + result["set_equality"] = set(predictions) == set(references) + except TypeError: + result["set_equality"] = None + return result + + @classmethod + def predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 4, 3]) + + @classmethod + def predictions_and_references_strings(cls): + return (["a", "b", "c", "d"], ["a", "b", "d", "c"]) + + @classmethod + def expected_results(cls): + return {"accuracy": 0.5, "set_equality": True} + + @classmethod + def other_predictions_and_references(cls): + return ([1, 3, 4, 5], [1, 2, 3, 4]) + + @classmethod + def other_expected_results(cls): + return {"accuracy": 0.25, "set_equality": False} + + @classmethod + def distributed_predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) + + @classmethod + def distributed_expected_results(cls): + return {"accuracy": 0.75, "set_equality": False} + + @classmethod + def separate_predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) + + @classmethod + def separate_expected_results(cls): + return [{"accuracy": 1.0, "set_equality": True}, {"accuracy": 0.5, "set_equality": False}] + + +class AnotherDummyMetric(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="another dummy metric for tests", + citation="insert citation here", + features=Features({"predictions": Value("int64"), "references": Value("int64")}), + ) + + def _compute(self, predictions, references): + return {"set_equality": False} + + @classmethod + def expected_results(cls): + return {"set_equality": False} + + +def properly_del_metric(metric): + """properly delete a metric on windows if the process is killed during multiprocessing""" + if metric is not None: + if metric.filelock is not None: + metric.filelock.release() + if metric.rendez_vous_lock is not None: + metric.rendez_vous_lock.release() + del metric.writer + del metric.data + del metric + + +def metric_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + time.sleep(wait) + results = metric.compute(predictions=preds, references=refs) + return results + finally: + properly_del_metric(metric) + + +def metric_add_batch_and_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + metric.add_batch(predictions=preds, references=refs) + time.sleep(wait) + results = metric.compute() + return results + finally: + properly_del_metric(metric) + + +def metric_add_and_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + time.sleep(wait) + results = metric.compute() + return results + finally: + properly_del_metric(metric) + + +class TestMetric(TestCase): + def test_dummy_metric(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + metric = DummyMetric(experiment_id="test_dummy_metric") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_dummy_metric") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_dummy_metric") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + # With keep_in_memory + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + self.assertDictEqual({}, metric.compute(predictions=[], references=[])) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + with self.assertRaisesRegex(ValueError, "Mismatch in the number"): + metric.add_batch(predictions=[1, 2, 3], references=[1, 2, 3, 4]) + del metric + + def test_metric_with_cache_dir(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + with tempfile.TemporaryDirectory() as tmp_dir: + metric = DummyMetric(experiment_id="test_dummy_metric", cache_dir=tmp_dir) + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + def test_concurrent_metrics(self): + preds, refs = DummyMetric.predictions_and_references() + other_preds, other_refs = DummyMetric.other_predictions_and_references() + expected_results = DummyMetric.expected_results() + other_expected_results = DummyMetric.other_expected_results() + + metric = DummyMetric(experiment_id="test_concurrent_metrics") + other_metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + self.assertDictEqual( + other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) + ) + del metric, other_metric + + metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + other_metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + metric.add_batch(predictions=preds, references=refs) + other_metric.add_batch(predictions=other_preds, references=other_refs) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + + for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): + metric.add(prediction=pred, reference=ref) + other_metric.add(prediction=other_pred, reference=other_ref) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + del metric, other_metric + + # With keep_in_memory + metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + self.assertDictEqual( + other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) + ) + + metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + metric.add_batch(predictions=preds, references=refs) + other_metric.add_batch(predictions=other_preds, references=other_refs) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + + for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): + metric.add(prediction=pred, reference=ref) + other_metric.add(prediction=other_pred, reference=other_ref) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + del metric, other_metric + + def test_separate_experiments_in_parallel(self): + with tempfile.TemporaryDirectory() as tmp_dir: + (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.separate_predictions_and_references() + expected_results = DummyMetric.separate_expected_results() + + pool = Pool(processes=2) + + results = pool.map( + metric_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + # more than one sec of waiting so that the second metric has to sample a new hashing name + results = pool.map( + metric_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 2), + (1, 0, preds_1, refs_1, None, tmp_dir, 2), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + results = pool.map( + metric_add_batch_and_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + def test_distributed_metrics(self): + with tempfile.TemporaryDirectory() as tmp_dir: + (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.distributed_predictions_and_references() + expected_results = DummyMetric.distributed_expected_results() + + pool = Pool(processes=4) + + results = pool.map( + metric_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0.5), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0.5), + (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_1", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_1", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_add_batch_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_2", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_2", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + # To use several distributed metrics on the same local file system, need to specify an experiment_id + try: + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), + (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), + ], + ) + except ValueError: + # We are fine with either raising a ValueError or computing well the metric + # Being sure we raise the error would means making the dummy dataset bigger + # and the test longer... + pass + else: + self.assertDictEqual(expected_results, results[0]) + self.assertDictEqual(expected_results, results[2]) + self.assertIsNone(results[1]) + self.assertIsNone(results[3]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "exp_0", tmp_dir, 0), + (2, 1, preds_1, refs_1, "exp_0", tmp_dir, 0), + (2, 0, preds_0, refs_0, "exp_1", tmp_dir, 0), + (2, 1, preds_1, refs_1, "exp_1", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertDictEqual(expected_results, results[2]) + self.assertIsNone(results[1]) + self.assertIsNone(results[3]) + del results + + # With keep_in_memory is not allowed + with self.assertRaises(ValueError): + DummyMetric( + experiment_id="test_distributed_metrics_4", + keep_in_memory=True, + num_process=2, + process_id=0, + cache_dir=tmp_dir, + ) + + def test_dummy_metric_pickle(self): + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_file = os.path.join(tmp_dir, "metric.pt") + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + metric = DummyMetric(experiment_id="test_dummy_metric_pickle") + + with open(tmp_file, "wb") as f: + pickle.dump(metric, f) + del metric + + with open(tmp_file, "rb") as f: + metric = pickle.load(f) + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + def test_input_numpy(self): + import numpy as np + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = np.array(preds), np.array(refs) + + metric = DummyMetric(experiment_id="test_input_numpy") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_numpy") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_numpy") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + @require_torch + def test_input_torch(self): + import torch + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = torch.tensor(preds), torch.tensor(refs) + + metric = DummyMetric(experiment_id="test_input_torch") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_torch") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_torch") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + @require_tf + def test_input_tf(self): + import tensorflow as tf + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = tf.constant(preds), tf.constant(refs) + + metric = DummyMetric(experiment_id="test_input_tf") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_tf") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_tf") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + def test_string_casting(self): + metric = DummyMetric(experiment_id="test_string_casting") + metric.info.features = Features({"predictions": Value("string"), "references": Value("string")}) + metric.compute(predictions=["a"], references=["a"]) + with self.assertRaises(ValueError): + metric.compute(predictions=[1], references=[1]) + + metric = DummyMetric(experiment_id="test_string_casting_2") + metric.info.features = Features( + {"predictions": Sequence(Value("string")), "references": Sequence(Value("string"))} + ) + metric.compute(predictions=[["a"]], references=[["a"]]) + with self.assertRaises(ValueError): + metric.compute(predictions=["a"], references=["a"]) + + def test_string_casting_tested_once(self): + + self.counter = 0 + + def checked_fct(fct): # wrapper function that increases a counter on each call + def wrapped(*args, **kwargs): + self.counter += 1 + return fct(*args, **kwargs) + + return wrapped + + with mock.patch( + "evaluate.EvaluationModule._enforce_nested_string_type", + checked_fct(DummyMetric._enforce_nested_string_type), + ): + metric = DummyMetric(experiment_id="test_string_casting_called_once") + metric.info.features = Features( + {"references": Sequence(Value("string")), "predictions": Sequence(Value("string"))} + ) + refs = [["test"] * 10] * 10 + preds = [["test"] * 10] * 10 + + metric.add_batch(references=refs, predictions=preds) + metric.add_batch(references=refs, predictions=preds) + + # the function is called twice for every batch's input: once on the + # sequence and then recursively agin on the first input of the sequence + self.assertEqual(self.counter, 8) + + def test_multiple_features(self): + metric = DummyMetric() + metric.info.features = [ + Features({"predictions": Value("int64"), "references": Value("int64")}), + Features({"predictions": Value("string"), "references": Value("string")}), + ] + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + + metric.info.features = [ + Features({"predictions": Value("string"), "references": Value("string")}), + Features({"predictions": Value("int64"), "references": Value("int64")}), + ] + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + + del metric + + +class MetricWithMultiLabel(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features( + {"predictions": Sequence(Value("int64")), "references": Sequence(Value("int64"))} + if self.config_name == "multilabel" + else {"predictions": Value("int64"), "references": Value("int64")} + ), + ) + + def _compute(self, predictions=None, references=None): + return ( + { + "accuracy": sum(i == j for i, j in zip(predictions, references)) / len(predictions), + } + if predictions + else {} + ) + + +@pytest.mark.parametrize( + "config_name, predictions, references, expected", + [ + (None, [1, 2, 3, 4], [1, 2, 4, 3], 0.5), # Multiclass: Value("int64") + ( + "multilabel", + [[1, 0], [1, 0], [1, 0], [1, 0]], + [[1, 0], [0, 1], [1, 1], [0, 0]], + 0.25, + ), # Multilabel: Sequence(Value("int64")) + ], +) +def test_metric_with_multilabel(config_name, predictions, references, expected, tmp_path): + cache_dir = tmp_path / "cache" + metric = MetricWithMultiLabel(config_name, cache_dir=cache_dir) + results = metric.compute(predictions=predictions, references=references) + assert results["accuracy"] == expected + + +def test_safety_checks_process_vars(): + with pytest.raises(ValueError): + _ = DummyMetric(process_id=-2) + + with pytest.raises(ValueError): + _ = DummyMetric(num_process=2, process_id=3) + + +class AccuracyWithNonStandardFeatureNames(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features({"inputs": Value("int64"), "targets": Value("int64")}), + ) + + def _compute(self, inputs, targets): + return ( + { + "accuracy": sum(i == j for i, j in zip(inputs, targets)) / len(targets), + } + if targets + else {} + ) + + @classmethod + def inputs_and_targets(cls): + return ([1, 2, 3, 4], [1, 2, 4, 3]) + + @classmethod + def expected_results(cls): + return {"accuracy": 0.5} + + +def test_metric_with_non_standard_feature_names_add(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + for input, target in zip(inputs, targets): + metric.add(inputs=input, targets=target) + results = metric.compute() + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +def test_metric_with_non_standard_feature_names_add_batch(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + metric.add_batch(inputs=inputs, targets=targets) + results = metric.compute() + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +def test_metric_with_non_standard_feature_names_compute(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + results = metric.compute(inputs=inputs, targets=targets) + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +class TestEvaluationcombined_evaluation(TestCase): + def test_single_module(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_add(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + for pred, ref in zip(preds, refs): + combined_evaluation.add(pred, ref) + self.assertDictEqual(expected_results, combined_evaluation.compute()) + + def test_add_batch(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + combined_evaluation.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, combined_evaluation.compute()) + + def test_force_prefix_with_dict(self): + prefix = "test_prefix" + preds, refs = DummyMetric.predictions_and_references() + + expected_results = DummyMetric.expected_results() + expected_results[f"{prefix}_accuracy"] = expected_results.pop("accuracy") + expected_results[f"{prefix}_set_equality"] = expected_results.pop("set_equality") + + combined_evaluation = combine({prefix: DummyMetric()}, force_prefix=True) + + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_duplicate_module(self): + preds, refs = DummyMetric.predictions_and_references() + dummy_metric = DummyMetric() + dummy_result = DummyMetric.expected_results() + combined_evaluation = combine([dummy_metric, dummy_metric]) + + expected_results = {} + for i in range(2): + for k in dummy_result: + expected_results[f"{dummy_metric.name}_{i}_{k}"] = dummy_result[k] + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_two_modules_with_same_score_name(self): + preds, refs = DummyMetric.predictions_and_references() + dummy_metric = DummyMetric() + another_dummy_metric = AnotherDummyMetric() + + dummy_result_1 = DummyMetric.expected_results() + dummy_result_2 = AnotherDummyMetric.expected_results() + + dummy_result_1[dummy_metric.name + "_set_equality"] = dummy_result_1.pop("set_equality") + dummy_result_1[another_dummy_metric.name + "_set_equality"] = dummy_result_2["set_equality"] + + combined_evaluation = combine([dummy_metric, another_dummy_metric]) + + self.assertDictEqual(dummy_result_1, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_modules_from_string(self): + expected_result = {"accuracy": 0.5, "recall": 0.5, "precision": 1.0} + predictions = [0, 1] + references = [1, 1] + + combined_evaluation = combine(["accuracy", "recall", "precision"]) + + self.assertDictEqual( + expected_result, combined_evaluation.compute(predictions=predictions, references=references) + ) + + def test_modules_from_string_poslabel(self): + expected_result = {"recall": 1.0, "precision": 0.5} + predictions = [0, 1, 0] + references = [1, 1, 0] + + combined_evaluation = combine(["recall", "precision"]) + + self.assertDictEqual( + expected_result, combined_evaluation.compute(predictions=predictions, references=references, pos_label=0) + ) diff --git a/ST/evaluate/tests/test_metric_common.py b/ST/evaluate/tests/test_metric_common.py new file mode 100644 index 0000000000000000000000000000000000000000..014dc0b32b5eb21808862301613360902bcacae1 --- /dev/null +++ b/ST/evaluate/tests/test_metric_common.py @@ -0,0 +1,227 @@ +# Copyright 2020 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import doctest +import glob +import importlib +import inspect +import os +import re +from contextlib import contextmanager +from functools import wraps +from unittest.mock import patch + +import numpy as np +import pytest +from absl.testing import parameterized + +import evaluate +from evaluate import load + +from .utils import _run_slow_tests, for_all_test_methods, local, slow + + +REQUIRE_FAIRSEQ = {"comet"} +_has_fairseq = importlib.util.find_spec("fairseq") is not None + +UNSUPPORTED_ON_WINDOWS = {"code_eval"} +_on_windows = os.name == "nt" + +SLOW_METRIC = {"perplexity", "regard", "toxicity"} + + +def skip_if_metric_requires_fairseq(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if not _has_fairseq and evaluation_module_name in REQUIRE_FAIRSEQ: + self.skipTest('"test requires Fairseq"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def skip_on_windows_if_not_windows_compatible(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if _on_windows and evaluation_module_name in UNSUPPORTED_ON_WINDOWS: + self.skipTest('"test not supported on Windows"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def skip_slow_metrics(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if not _run_slow_tests and evaluation_module_name in SLOW_METRIC: + self.skipTest('"test is slow"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def get_local_module_names(): + metrics = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./metrics/*/")] + comparisons = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./comparisons/*/")] + measurements = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./measurements/*/")] + + evaluation_modules = metrics + comparisons + measurements + evaluation_module_types = ( + ["metric"] * len(metrics) + ["comparison"] * len(comparisons) + ["measurement"] * len(measurements) + ) + + return [ + {"testcase_name": f"{t}_{x}", "evaluation_module_name": x, "evaluation_module_type": t} + for x, t in zip(evaluation_modules, evaluation_module_types) + if x != "gleu" # gleu is unfinished + ] + + +@parameterized.named_parameters(get_local_module_names()) +@for_all_test_methods(skip_if_metric_requires_fairseq, skip_on_windows_if_not_windows_compatible, skip_slow_metrics) +@local +class LocalModuleTest(parameterized.TestCase): + INTENSIVE_CALLS_PATCHER = {} + evaluation_module_name = None + evaluation_module_type = None + + def test_load(self, evaluation_module_name, evaluation_module_type): + doctest.ELLIPSIS_MARKER = "[...]" + evaluation_module = importlib.import_module( + evaluate.loading.evaluation_module_factory( + os.path.join(evaluation_module_type + "s", evaluation_module_name), module_type=evaluation_module_type + ).module_path + ) + evaluation_instance = evaluate.loading.import_main_class(evaluation_module.__name__) + # check parameters + parameters = inspect.signature(evaluation_instance._compute).parameters + self.assertTrue(all([p.kind != p.VAR_KEYWORD for p in parameters.values()])) # no **kwargs + # run doctest + with self.patch_intensive_calls(evaluation_module_name, evaluation_module.__name__): + with self.use_local_metrics(evaluation_module_type): + try: + results = doctest.testmod(evaluation_module, verbose=True, raise_on_error=True) + except doctest.UnexpectedException as e: + raise e.exc_info[1] # raise the exception that doctest caught + self.assertEqual(results.failed, 0) + self.assertGreater(results.attempted, 1) + + @slow + def test_load_real_metric(self, evaluation_module_name, evaluation_module_type): + doctest.ELLIPSIS_MARKER = "[...]" + metric_module = importlib.import_module( + evaluate.loading.evaluation_module_factory( + os.path.join(evaluation_module_type, evaluation_module_name) + ).module_path + ) + # run doctest + with self.use_local_metrics(): + results = doctest.testmod(metric_module, verbose=True, raise_on_error=True) + self.assertEqual(results.failed, 0) + self.assertGreater(results.attempted, 1) + + @contextmanager + def patch_intensive_calls(self, evaluation_module_name, module_name): + if evaluation_module_name in self.INTENSIVE_CALLS_PATCHER: + with self.INTENSIVE_CALLS_PATCHER[evaluation_module_name](module_name): + yield + else: + yield + + @contextmanager + def use_local_metrics(self, evaluation_module_type): + def load_local_metric(evaluation_module_name, *args, **kwargs): + return load(os.path.join(evaluation_module_type + "s", evaluation_module_name), *args, **kwargs) + + with patch("evaluate.load") as mock_load: + mock_load.side_effect = load_local_metric + yield + + @classmethod + def register_intensive_calls_patcher(cls, evaluation_module_name): + def wrapper(patcher): + patcher = contextmanager(patcher) + cls.INTENSIVE_CALLS_PATCHER[evaluation_module_name] = patcher + return patcher + + return wrapper + + +# Metrics intensive calls patchers +# -------------------------------- + + +@LocalModuleTest.register_intensive_calls_patcher("bleurt") +def patch_bleurt(module_name): + import tensorflow.compat.v1 as tf + from bleurt.score import Predictor + + tf.flags.DEFINE_string("sv", "", "") # handle pytest cli flags + + class MockedPredictor(Predictor): + def predict(self, input_dict): + assert len(input_dict["input_ids"]) == 2 + return np.array([1.03, 1.04]) + + # mock predict_fn which is supposed to do a forward pass with a bleurt model + with patch("bleurt.score._create_predictor") as mock_create_predictor: + mock_create_predictor.return_value = MockedPredictor() + yield + + +@LocalModuleTest.register_intensive_calls_patcher("bertscore") +def patch_bertscore(module_name): + import torch + + def bert_cos_score_idf(model, refs, *args, **kwargs): + return torch.tensor([[1.0, 1.0, 1.0]] * len(refs)) + + # mock get_model which is supposed to do download a bert model + # mock bert_cos_score_idf which is supposed to do a forward pass with a bert model + with patch("bert_score.scorer.get_model"), patch( + "bert_score.scorer.bert_cos_score_idf" + ) as mock_bert_cos_score_idf: + mock_bert_cos_score_idf.side_effect = bert_cos_score_idf + yield + + +@LocalModuleTest.register_intensive_calls_patcher("comet") +def patch_comet(module_name): + def load_from_checkpoint(model_path): + class Model: + def predict(self, data, *args, **kwargs): + assert len(data) == 2 + scores = [0.19, 0.92] + return scores, sum(scores) / len(scores) + + return Model() + + # mock load_from_checkpoint which is supposed to do download a bert model + # mock load_from_checkpoint which is supposed to do download a bert model + with patch("comet.download_model") as mock_download_model: + mock_download_model.return_value = None + with patch("comet.load_from_checkpoint") as mock_load_from_checkpoint: + mock_load_from_checkpoint.side_effect = load_from_checkpoint + yield + + +def test_seqeval_raises_when_incorrect_scheme(): + metric = load(os.path.join("metrics", "seqeval")) + wrong_scheme = "ERROR" + error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}" + with pytest.raises(ValueError, match=re.escape(error_message)): + metric.compute(predictions=[], references=[], scheme=wrong_scheme) diff --git a/ST/evaluate/tests/test_save.py b/ST/evaluate/tests/test_save.py new file mode 100644 index 0000000000000000000000000000000000000000..adca53feedec431c17efa85bbecc0f71e21d761e --- /dev/null +++ b/ST/evaluate/tests/test_save.py @@ -0,0 +1,44 @@ +import json +import shutil +import tempfile +from pathlib import Path +from unittest import TestCase + +import evaluate + + +result_dict = {"metric": 1.0, "model_name": "x"} + +SAVE_EXTRA_KEYS = ["_timestamp", "_git_commit_hash", "_evaluate_version", "_python_version", "_interpreter_path"] + + +class TestSave(TestCase): + def setUp(self): + self.save_path = Path(tempfile.mkdtemp()) + + def tearDown(self): + shutil.rmtree(self.save_path) + + def test_save_to_folder(self): + file_path = evaluate.save(self.save_path, **result_dict) + with open(file_path, "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) + + def test_save_to_folder_nested(self): + file_path = evaluate.save(self.save_path / "sub_dir1/sub_dir2", **result_dict) + with open(file_path, "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) + + def test_save_to_file(self): + _ = evaluate.save(self.save_path / "test.json", **result_dict) + with open(self.save_path / "test.json", "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) diff --git a/ST/evaluate/tests/test_trainer_evaluator_parity.py b/ST/evaluate/tests/test_trainer_evaluator_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..b513da88663088e18899e770274f102a7c06e546 --- /dev/null +++ b/ST/evaluate/tests/test_trainer_evaluator_parity.py @@ -0,0 +1,313 @@ +import json +import os +import shutil +import subprocess +import tempfile +import unittest + +import numpy as np +import torch +import transformers +from datasets import load_dataset +from transformers import AutoFeatureExtractor, AutoModelForImageClassification, Trainer, TrainingArguments, pipeline + +from evaluate import evaluator, load + +from .utils import slow + + +class TestEvaluatorTrainerParity(unittest.TestCase): + def setUp(self): + self.dir_path = tempfile.mkdtemp("evaluator_trainer_parity_test") + + transformers_version = transformers.__version__ + branch = "" + if not transformers_version.endswith(".dev0"): + branch = f"--branch v{transformers_version}" + subprocess.run( + f"git clone --depth 3 --filter=blob:none --sparse {branch} https://github.com/huggingface/transformers", + shell=True, + cwd=self.dir_path, + ) + + def tearDown(self): + shutil.rmtree(self.dir_path, ignore_errors=True) + + def test_text_classification_parity(self): + model_name = "philschmid/tiny-bert-sst2-distilled" + + subprocess.run( + "git sparse-checkout set examples/pytorch/text-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/text-classification/run_glue.py" + f" --model_name_or_path {model_name}" + f" --task_name sst2" + f" --do_eval" + f" --max_seq_length 9999999999" # rely on tokenizer.model_max_length for max_length + f" --output_dir {os.path.join(self.dir_path, 'textclassification_sst2_transformers')}" + f" --max_eval_samples 80", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'textclassification_sst2_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("glue", "sst2", split="validation[:80]") + + pipe = pipeline(task="text-classification", model=model_name, tokenizer=model_name) + + task_evaluator = evaluator(task="text-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="sentence", + label_column="label", + label_mapping={"negative": 0, "positive": 1}, + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + @slow + def test_text_classification_parity_two_columns(self): + model_name = "prajjwal1/bert-tiny-mnli" + max_eval_samples = 150 + + subprocess.run( + "git sparse-checkout set examples/pytorch/text-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/text-classification/run_glue.py" + f" --model_name_or_path {model_name}" + f" --task_name mnli" + f" --do_eval" + f" --max_seq_length 256" + f" --output_dir {os.path.join(self.dir_path, 'textclassification_mnli_transformers')}" + f" --max_eval_samples {max_eval_samples}", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'textclassification_mnli_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("glue", "mnli", split=f"validation_matched[:{max_eval_samples}]") + + pipe = pipeline(task="text-classification", model=model_name, tokenizer=model_name, max_length=256) + + task_evaluator = evaluator(task="text-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="premise", + second_input_column="hypothesis", + label_column="label", + label_mapping={"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}, + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + def test_image_classification_parity(self): + # we can not compare to the Pytorch transformers example, that uses custom preprocessing on the images + model_name = "douwekiela/resnet-18-finetuned-dogfood" + dataset_name = "beans" + max_eval_samples = 120 + + raw_dataset = load_dataset(dataset_name, split="validation") + eval_dataset = raw_dataset.select(range(max_eval_samples)) + + feature_extractor = AutoFeatureExtractor.from_pretrained(model_name) + model = AutoModelForImageClassification.from_pretrained(model_name) + + def collate_fn(examples): + pixel_values = torch.stack( + [torch.tensor(feature_extractor(example["image"])["pixel_values"][0]) for example in examples] + ) + labels = torch.tensor([example["labels"] for example in examples]) + return {"pixel_values": pixel_values, "labels": labels} + + metric = load("accuracy") + trainer = Trainer( + model=model, + args=TrainingArguments( + output_dir=os.path.join(self.dir_path, "imageclassification_beans_transformers"), + remove_unused_columns=False, + ), + train_dataset=None, + eval_dataset=eval_dataset, + compute_metrics=lambda p: metric.compute( + predictions=np.argmax(p.predictions, axis=1), references=p.label_ids + ), + tokenizer=None, + data_collator=collate_fn, + ) + + metrics = trainer.evaluate() + trainer.save_metrics("eval", metrics) + + with open( + f"{os.path.join(self.dir_path, 'imageclassification_beans_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + pipe = pipeline(task="image-classification", model=model_name, feature_extractor=model_name) + + task_evaluator = evaluator(task="image-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="image", + label_column="labels", + label_mapping=model.config.label2id, + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + def test_question_answering_parity(self): + model_name_v1 = "anas-awadalla/bert-tiny-finetuned-squad" + model_name_v2 = "mrm8488/bert-tiny-finetuned-squadv2" + + subprocess.run( + "git sparse-checkout set examples/pytorch/question-answering", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + # test squad_v1-like dataset + subprocess.run( + f"python examples/pytorch/question-answering/run_qa.py" + f" --model_name_or_path {model_name_v1}" + f" --dataset_name squad" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'questionanswering_squad_transformers')}" + f" --max_eval_samples 100" + f" --max_seq_length 384", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'questionanswering_squad_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("squad", split="validation[:100]") + + pipe = pipeline( + task="question-answering", + model=model_name_v1, + tokenizer=model_name_v1, + max_answer_len=30, + padding="max_length", + ) + + task_evaluator = evaluator(task="question-answering") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="squad", + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_f1"], evaluator_results["f1"]) + self.assertEqual(transformers_results["eval_exact_match"], evaluator_results["exact_match"]) + + # test squad_v2-like dataset + subprocess.run( + f"python examples/pytorch/question-answering/run_qa.py" + f" --model_name_or_path {model_name_v2}" + f" --dataset_name squad_v2" + f" --version_2_with_negative" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'questionanswering_squadv2_transformers')}" + f" --max_eval_samples 100" + f" --max_seq_length 384", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'questionanswering_squadv2_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("squad_v2", split="validation[:100]") + + pipe = pipeline( + task="question-answering", + model=model_name_v2, + tokenizer=model_name_v2, + max_answer_len=30, + ) + + task_evaluator = evaluator(task="question-answering") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="squad_v2", + strategy="simple", + squad_v2_format=True, + ) + + self.assertEqual(transformers_results["eval_f1"], evaluator_results["f1"]) + self.assertEqual(transformers_results["eval_HasAns_f1"], evaluator_results["HasAns_f1"]) + self.assertEqual(transformers_results["eval_NoAns_f1"], evaluator_results["NoAns_f1"]) + + def test_token_classification_parity(self): + model_name = "hf-internal-testing/tiny-bert-for-token-classification" + n_samples = 500 + + subprocess.run( + "git sparse-checkout set examples/pytorch/token-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/token-classification/run_ner.py" + f" --model_name_or_path {model_name}" + f" --dataset_name conll2003" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'tokenclassification_conll2003_transformers')}" + f" --max_eval_samples {n_samples}", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + os.path.join(self.dir_path, "tokenclassification_conll2003_transformers", "eval_results.json"), "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("conll2003", split=f"validation[:{n_samples}]") + + pipe = pipeline(task="token-classification", model=model_name) + + e = evaluator(task="token-classification") + evaluator_results = e.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="seqeval", + input_column="tokens", + label_column="ner_tags", + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["overall_accuracy"]) + self.assertEqual(transformers_results["eval_f1"], evaluator_results["overall_f1"]) diff --git a/ST/evaluate/tests/test_viz.py b/ST/evaluate/tests/test_viz.py new file mode 100644 index 0000000000000000000000000000000000000000..c44e5104e377fba28a067337c89991d0df52e5b3 --- /dev/null +++ b/ST/evaluate/tests/test_viz.py @@ -0,0 +1,24 @@ +from unittest import TestCase + +import matplotlib.pyplot as plt + +from evaluate.visualization import radar_plot + + +class TestViz(TestCase): + def test_invert_range(self): + data = [{"accuracy": 0.9, "precision": 0.8}, {"accuracy": 0.7, "precision": 0.6}] + model_names = ["model1", "model2"] + wrong_invert_range = ["latency_in_seconds"] # Value not present in data + with self.assertRaises(ValueError): + radar_plot(data, model_names, wrong_invert_range) + + def test_output_is_plot(self): + data = [ + {"accuracy": 0.9, "precision": 0.8, "latency_in_seconds": 48.1}, + {"accuracy": 0.7, "precision": 0.6, "latency_in_seconds": 51.4}, + ] + model_names = ["model1", "model2"] + invert_range = ["latency_in_seconds"] + out_plt = radar_plot(data, model_names, invert_range) + self.assertIsInstance(out_plt, plt.Figure) diff --git a/ST/evaluate/tests/utils.py b/ST/evaluate/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc27d9dbbb56b5d39e3b63e54237f13bf4ffacfd --- /dev/null +++ b/ST/evaluate/tests/utils.py @@ -0,0 +1,290 @@ +import os +import tempfile +import unittest +from contextlib import contextmanager +from copy import deepcopy +from distutils.util import strtobool +from enum import Enum +from pathlib import Path +from unittest.mock import patch + +from evaluate import config + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError(f"If set, {key} must be yes or no.") + return _value + + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_remote_tests = parse_flag_from_env("RUN_REMOTE", default=False) +_run_local_tests = parse_flag_from_env("RUN_LOCAL", default=True) +_run_packaged_tests = parse_flag_from_env("RUN_PACKAGED", default=True) + + +def require_beam(test_case): + """ + Decorator marking a test that requires Apache Beam. + + These tests are skipped when Apache Beam isn't installed. + + """ + if not config.TORCH_AVAILABLE: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_faiss(test_case): + """ + Decorator marking a test that requires Faiss. + + These tests are skipped when Faiss isn't installed. + + """ + try: + import faiss # noqa + except ImportError: + test_case = unittest.skip("test requires faiss")(test_case) + return test_case + + +def require_regex(test_case): + """ + Decorator marking a test that requires regex. + + These tests are skipped when Regex isn't installed. + + """ + try: + import regex # noqa + except ImportError: + test_case = unittest.skip("test requires regex")(test_case) + return test_case + + +def require_elasticsearch(test_case): + """ + Decorator marking a test that requires ElasticSearch. + + These tests are skipped when ElasticSearch isn't installed. + + """ + try: + import elasticsearch # noqa + except ImportError: + test_case = unittest.skip("test requires elasticsearch")(test_case) + return test_case + + +def require_torch(test_case): + """ + Decorator marking a test that requires PyTorch. + + These tests are skipped when PyTorch isn't installed. + + """ + if not config.TORCH_AVAILABLE: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_tf(test_case): + """ + Decorator marking a test that requires TensorFlow. + + These tests are skipped when TensorFlow isn't installed. + + """ + if not config.TF_AVAILABLE: + test_case = unittest.skip("test requires TensorFlow")(test_case) + return test_case + + +def require_jax(test_case): + """ + Decorator marking a test that requires JAX. + + These tests are skipped when JAX isn't installed. + + """ + if not config.JAX_AVAILABLE: + test_case = unittest.skip("test requires JAX")(test_case) + return test_case + + +def require_pil(test_case): + """ + Decorator marking a test that requires Pillow. + + These tests are skipped when Pillow isn't installed. + + """ + if not config.PIL_AVAILABLE: + test_case = unittest.skip("test requires Pillow")(test_case) + return test_case + + +def require_transformers(test_case): + """ + Decorator marking a test that requires transformers. + + These tests are skipped when transformers isn't installed. + + """ + try: + import transformers # noqa F401 + except ImportError: + return unittest.skip("test requires transformers")(test_case) + else: + return test_case + + +def slow(test_case): + """ + Decorator marking a test as slow. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable + to a truthy value to run them. + + """ + if not _run_slow_tests or _run_slow_tests == 0: + test_case = unittest.skip("test is slow")(test_case) + return test_case + + +def local(test_case): + """ + Decorator marking a test as local + + Local tests are run by default. Set the RUN_LOCAL environment variable + to a falsy value to not run them. + """ + if not _run_local_tests or _run_local_tests == 0: + test_case = unittest.skip("test is local")(test_case) + return test_case + + +def packaged(test_case): + """ + Decorator marking a test as packaged + + Packaged tests are run by default. Set the RUN_PACKAGED environment variable + to a falsy value to not run them. + """ + if not _run_packaged_tests or _run_packaged_tests == 0: + test_case = unittest.skip("test is packaged")(test_case) + return test_case + + +def remote(test_case): + """ + Decorator marking a test as one that relies on GitHub or the Hugging Face Hub. + + Remote tests are skipped by default. Set the RUN_REMOTE environment variable + to a falsy value to not run them. + """ + if not _run_remote_tests or _run_remote_tests == 0: + test_case = unittest.skip("test requires remote")(test_case) + return test_case + + +def for_all_test_methods(*decorators): + def decorate(cls): + for name, fn in cls.__dict__.items(): + if callable(fn) and name.startswith("test"): + for decorator in decorators: + fn = decorator(fn) + setattr(cls, name, fn) + return cls + + return decorate + + +class RequestWouldHangIndefinitelyError(Exception): + pass + + +class OfflineSimulationMode(Enum): + CONNECTION_FAILS = 0 + CONNECTION_TIMES_OUT = 1 + HF_EVALUATE_OFFLINE_SET_TO_1 = 2 + + +@contextmanager +def offline(mode=OfflineSimulationMode.CONNECTION_FAILS, timeout=1e-16): + """ + Simulate offline mode. + + There are three offline simulatiom modes: + + CONNECTION_FAILS (default mode): a ConnectionError is raised for each network call. + Connection errors are created by mocking socket.socket + CONNECTION_TIMES_OUT: the connection hangs until it times out. + The default timeout value is low (1e-16) to speed up the tests. + Timeout errors are created by mocking requests.request + HF_EVALUATE_OFFLINE_SET_TO_1: the HF_EVALUATE_OFFLINE environment variable is set to 1. + This makes the http/ftp calls of the library instantly fail and raise an OfflineModeEmabled error. + """ + from requests import request as online_request + + def timeout_request(method, url, **kwargs): + # Change the url to an invalid url so that the connection hangs + invalid_url = "https://10.255.255.1" + if kwargs.get("timeout") is None: + raise RequestWouldHangIndefinitelyError( + f"Tried a call to {url} in offline mode with no timeout set. Please set a timeout." + ) + kwargs["timeout"] = timeout + try: + return online_request(method, invalid_url, **kwargs) + except Exception as e: + # The following changes in the error are just here to make the offline timeout error prettier + e.request.url = url + max_retry_error = e.args[0] + max_retry_error.args = (max_retry_error.args[0].replace("10.255.255.1", f"OfflineMock[{url}]"),) + e.args = (max_retry_error,) + raise + + def offline_socket(*args, **kwargs): + raise OSError("Offline mode is enabled.") + + if mode is OfflineSimulationMode.CONNECTION_FAILS: + # inspired from https://stackoverflow.com/a/18601897 + with patch("socket.socket", offline_socket): + yield + elif mode is OfflineSimulationMode.CONNECTION_TIMES_OUT: + # inspired from https://stackoverflow.com/a/904609 + with patch("requests.request", timeout_request): + with patch("requests.api.request", timeout_request): + yield + elif mode is OfflineSimulationMode.HF_EVALUATE_OFFLINE_SET_TO_1: + with patch("evaluate.config.HF_EVALUATE_OFFLINE", True): + yield + else: + raise ValueError("Please use a value from the OfflineSimulationMode enum.") + + +@contextmanager +def set_current_working_directory_to_temp_dir(*args, **kwargs): + original_working_dir = str(Path().resolve()) + with tempfile.TemporaryDirectory(*args, **kwargs) as tmp_dir: + try: + os.chdir(tmp_dir) + yield + finally: + os.chdir(original_working_dir) + + +def is_rng_equal(rng1, rng2): + return deepcopy(rng1).integers(0, 100, 10).tolist() == deepcopy(rng2).integers(0, 100, 10).tolist() diff --git a/ST/inference/demo.py b/ST/inference/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..32b1431916a4e26210c46645dc8ad100c811b545 --- /dev/null +++ b/ST/inference/demo.py @@ -0,0 +1,144 @@ +import requests +import gradio as gr +import soundfile as sf +import time + + + +def speech_translation(audio, language): + if audio is None: + return "No audio input provided!", "No audio input provided!" + + # Convert audio to .wav format if not already + if not audio.endswith(".wav"): + wav_data, samplerate = sf.read(audio) + sf.write("temp_audio.wav", wav_data, samplerate) + audio_file = "temp_audio.wav" + else: + audio_file = audio + + # ASR processing + files = { + 'file': open(audio_file, "rb"), + 'language': (None, language), + 'vtt': (None, 'true'), + } + response = requests.post('https://asr.iitm.ac.in/ssl_asr/decode', files=files) + + print(response.json()) + try: + asr_output = response.json()['transcript'] + except: + asr_output = "Error in ASR processing" + + asr_output = asr_output.replace("।", "") + asr_output = asr_output.replace(".", "") + + time.sleep(1) + + + if language == "telugu": + lang = "te" + elif language == "hindi": + lang = "hi" + elif language == "marathi": + lang = "mr" + elif language == "bengali": + lang = "bn" + + payload = { + "pipelineTasks": [ + { + "taskType": "translation", + "config": { + "language": { + "sourceLanguage": lang, + "targetLanguage": "en", + }, + }, + + } + ], + "pipelineRequestConfig": { + "pipelineId" : "64392f96daac500b55c543cd" + + } + } + headers = { + "Content-Type": "application/json", + "userID": "2aeef589f4584eb08aa0b9c49761aeb8", + "ulcaApiKey": "02ed10445a-66b0-4061-9030-9b0b8b37a4f1" + } + + response = requests.post('https://meity-auth.ulcacontrib.org/ulca/apis/v0/model/getModelsPipeline', json=payload, headers=headers) + + + if response.status_code == 200: + response_data = response.json() + print(response_data) + service_id = response_data["pipelineResponseConfig"][0]["config"][0]["serviceId"] + # if lang=="te": + # service_id = "bhashini/iitm/asr-dravidian--gpu--t4" + # else: + # service_id = "bhashini/iitm/asr-indoaryan--gpu--t4" + + + # print("halfway") + + + + compute_payload = { + "pipelineTasks": [ + { + "taskType": "translation", + "config": { + "language": { + "sourceLanguage": lang, + "targetLanguage": "en", + }, + }, + + } + ], + "inputData": {"input": [{"source": asr_output}]}, + } + + callback_url = response_data["pipelineInferenceAPIEndPoint"]["callbackUrl"] + + headers2 = { + "Content-Type": "application/json", + response_data["pipelineInferenceAPIEndPoint"]["inferenceApiKey"]["name"]: + response_data["pipelineInferenceAPIEndPoint"]["inferenceApiKey"]["value"] + } + + compute_response = requests.post(callback_url, json=compute_payload, headers=headers2) + # print(compute_response.json()) + + if compute_response.status_code == 200: + compute_response_data = compute_response.json() + print(compute_response_data) + translated_content = compute_response_data["pipelineResponse"][0]["output"][0]["target"] + print( + + "Translation successful", + translated_content + ) + + else: + print ( + "status_code", compute_response.status_code) + + return translated_content + +iface = gr.Interface( + fn=speech_translation, + inputs=[ + gr.Audio(type="filepath", label="Record your speech"), + gr.Dropdown(["telugu", "hindi", "marathi", "bengali"], label="Select Language") + ], + outputs=["text"], + title="Speech Translation", + description="Record your speech and get the English translation.", +) + +iface.launch() \ No newline at end of file diff --git a/ST/inference/demo_streamlit.py b/ST/inference/demo_streamlit.py new file mode 100644 index 0000000000000000000000000000000000000000..52200cf332fc7679577b358066a51139e4d8ddb9 --- /dev/null +++ b/ST/inference/demo_streamlit.py @@ -0,0 +1,179 @@ +import requests +import soundfile as sf +import time +import streamlit as st + +def speech_translation(audio_file_path, language): + if audio_file_path is None: + return "No audio input provided!" + + # Convert audio to .wav format if not already + if not audio_file_path.endswith(".wav"): + wav_data, samplerate = sf.read(audio_file_path) + sf.write("temp_audio.wav", wav_data, samplerate) + audio_file_path = "temp_audio.wav" + else: + audio_file_path = audio_file_path + + # ASR processing + files = { + 'file': open(audio_file_path, "rb"), + 'language': (None, language), + 'vtt': (None, 'true'), + } + response = requests.post('https://asr.iitm.ac.in/ssl_asr/decode', files=files) + + print(response.json()) + try: + asr_output = response.json()['transcript'] + except: + return "Error in ASR processing" + + asr_output = asr_output.replace("।", "") + asr_output = asr_output.replace(".", "") + + time.sleep(1) + + lang = "" + if language == "telugu": + lang = "te" + elif language == "hindi": + lang = "hi" + elif language == "marathi": + lang = "mr" + elif language == "bengali": + lang = "bn" + + payload = { + "pipelineTasks": [ + { + "taskType": "translation", + "config": { + "language": { + "sourceLanguage": lang, + "targetLanguage": "en", + }, + }, + } + ], + "pipelineRequestConfig": { + "pipelineId": "64392f96daac500b55c543cd" + } + } + headers = { + "Content-Type": "application/json", + "userID": "2aeef589f4584eb08aa0b9c49761aeb8", + "ulcaApiKey": "02ed10445a-66b0-4061-9030-9b0b8b37a4f1" + } + + response = requests.post('https://meity-auth.ulcacontrib.org/ulca/apis/v0/model/getModelsPipeline', json=payload, headers=headers) + + if response.status_code == 200: + response_data = response.json() + print(response_data) + + compute_payload = { + "pipelineTasks": [ + { + "taskType": "translation", + "config": { + "language": { + "sourceLanguage": lang, + "targetLanguage": "en", + }, + }, + } + ], + "inputData": {"input": [{"source": asr_output}]}, + } + + callback_url = response_data["pipelineInferenceAPIEndPoint"]["callbackUrl"] + + headers2 = { + "Content-Type": "application/json", + response_data["pipelineInferenceAPIEndPoint"]["inferenceApiKey"]["name"]: + response_data["pipelineInferenceAPIEndPoint"]["inferenceApiKey"]["value"] + } + + compute_response = requests.post(callback_url, json=compute_payload, headers=headers2) + + if compute_response.status_code == 200: + compute_response_data = compute_response.json() + print(compute_response_data) + translated_content = compute_response_data["pipelineResponse"][0]["output"][0]["target"] + return translated_content + else: + return f"Error in translation: status code {compute_response.status_code}" + else: + return f"Error in fetching model pipeline: status code {response.status_code}" + + return "Translation failed" + +# Streamlit UI +st.title("Speech Translation") +st.write("Record your speech and get the English translation.") + +# Audio Recorder HTML +st.markdown(""" +

Record Audio

+ + + + +""", unsafe_allow_html=True) + +uploaded_file = st.file_uploader("Upload an audio file", type=["wav", "mp3"]) +language = st.selectbox("Select Language", ["telugu", "hindi", "marathi", "bengali"]) + +if st.button("Translate"): + if uploaded_file is not None: + with open("uploaded_audio.wav", "wb") as f: + f.write(uploaded_file.getbuffer()) + result = speech_translation("uploaded_audio.wav", language) + st.text_area("Translation", result) + elif st.session_state.get('recorded_audio'): + result = speech_translation(st.session_state['recorded_audio'], language) + st.text_area("Translation", result) + else: + st.write("Please upload an audio file or record your speech and select a language.") \ No newline at end of file diff --git a/ST/inference/inference1.py b/ST/inference/inference1.py new file mode 100644 index 0000000000000000000000000000000000000000..f90d57297dceecda4f15046a079f11fd2bcd81cd --- /dev/null +++ b/ST/inference/inference1.py @@ -0,0 +1,53 @@ +import torch +from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline +from datasets import load_dataset +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "6" # SET the GPUs you want to use +import csv +from transformers import WhisperProcessor, WhisperForConditionalGeneration + +# device = "cuda:0" if torch.cuda.is_available() else "cpu" +torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + +model_id = "openai/whisper-large-v3" + +model = WhisperForConditionalGeneration.from_pretrained(model_id) +processor = WhisperProcessor.from_pretrained(model_id) + +mp3_folder = "./extra_epi_wav/" + +# Get a list of all the mp3 files in the folder +mp3_files = [file for file in os.listdir(mp3_folder) if file.endswith(".wav")] + +# Create a CSV file to store the transcripts +csv_filename = "transcripts_with_lang.csv" + +with open(csv_filename, mode='w', newline='', encoding='utf-8') as csv_file: + fieldnames = ['File Name', 'Transcript', 'lang'] + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + + # Write the header to the CSV file + writer.writeheader() + + # Process each mp3 file and write the results to the CSV file + processed_files_counter = 0 + for mp3_file in mp3_files: + mp3_path = os.path.join(mp3_folder, mp3_file) + result = pipe(mp3_path) + print(result) + + transcript = result["text"] + # p + + processed_files_counter += 1 + + # Check progress after every 10 files + if processed_files_counter % 10 == 0: + print(f"{processed_files_counter} files processed.") + + # Write the file name and transcript to the CSV file + writer.writerow({'File Name': mp3_file, 'Transcript': transcript}) + +print(f"Transcripts saved to {csv_filename}") + + diff --git a/ST/inference/inference_fast.py b/ST/inference/inference_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..5abeb1e7bcf37cb57aa1f7e8cda295ff849ba637 --- /dev/null +++ b/ST/inference/inference_fast.py @@ -0,0 +1,77 @@ +import torch +from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline +from datasets import load_dataset +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "0" # SET the GPUs you want to use +import csv + + +device = "cuda:0" if torch.cuda.is_available() else "cpu" +print(device) +torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + +model_id = "openai/whisper-large-v3" + +model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True +) +model.to(device) + +processor = AutoProcessor.from_pretrained(model_id) + + +pipe = pipeline( + "automatic-speech-recognition", + model=model, + tokenizer=processor.tokenizer, + feature_extractor=processor.feature_extractor, + max_new_tokens=128, + chunk_length_s=30, + batch_size=16, + return_timestamps=True, + return_language=True, + torch_dtype=torch_dtype, + device=device, +) + +# Specify the folder containing the mp3 files +mp3_folder = "./eng_audio/" + +# Get a list of all the mp3 files in the folder +mp3_files = [file for file in os.listdir(mp3_folder) if file.endswith(".mp3")] +# mp3_files = ["p2_17.wav"] +# Create a CSV file to store the transcripts +csv_filename = "transcripts_english.csv" + +with open(csv_filename, mode='a', newline='', encoding='utf-8') as csv_file: + fieldnames = ['File Name', 'Transcript', 'Language'] + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + + # Write the header to the CSV file + # writer.writeheader() + + # Process each mp3 file and write the results to the CSV file + processed_files_counter = 0 + for mp3_file in mp3_files: + mp3_path = os.path.join(mp3_folder, mp3_file) + save_filename = "tmp.wav" + cmd = f"ffmpeg -i {mp3_path} -ac 1 -ar 16000 {save_filename} -y -hide_banner -loglevel error" + os.system(cmd) + mp3_path = save_filename + + result = pipe(mp3_path,generate_kwargs={"language": "english"}) + + transcript = result["text"].strip() + lang = result["chunks"][0]["language"] + + + processed_files_counter += 1 + + # Check progress after every 10 files + if processed_files_counter % 10 == 0: + print(f"{processed_files_counter} files processed.") + + # Write the file name and transcript to the CSV file + writer.writerow({'File Name': mp3_file, 'Transcript': transcript, 'Language': lang}) + +print(f"Transcripts saved to {csv_filename}") diff --git a/ST/inference/inference_req_hf.py b/ST/inference/inference_req_hf.py new file mode 100644 index 0000000000000000000000000000000000000000..c4f7d0166fad437c4b36e61f846e71aa0e3afd8e --- /dev/null +++ b/ST/inference/inference_req_hf.py @@ -0,0 +1,88 @@ +import os +import numpy as np +import pandas as pd +import whisper +# import torchaudio +# import librosa +import sys +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "2" + +from tqdm.notebook import tqdm +# from whisper.normalizers import EnglishTextNormalizer + +class Decoder: + """ + Class to perform ASR predictions + """ + + def __init__(self, model_type, language='mr'): + """Initialization of the class + More details on the whisper model and its types can be found here: https://github.com/openai/whisper + Convert HF model to openai-whisper: https://github.com/openai/whisper/discussions/830 + Args: + model_type (str): Should be one of 'tiny', 'base', 'small', 'medium', 'large', 'large-v2' + """ + + assert model_type in ['tiny', 'base', 'small', 'medium', 'large', 'large-v2', 'large-v3'], "Wrong model type" + print('Info: Loading model') + self.model = whisper.load_model(model_type) + + self.decode_options = whisper.DecodingOptions(language=language, without_timestamps=True) + self.device = "cuda" # if torch.cuda.is_available() else "cpu" + print("Info: Initialization done") + + def decode(self, filepath): + """Get the transcription(in hindi) for the audio file + + Args: + filepath (str): Absolute path of the audio file + + Returns: + str: transcription of the audio in hindi + """ + print() + + result = self.model.transcribe(filepath, language="mr", verbose=False, without_timestamps=True, fp16=False) + + return result["text"] + +if __name__ == "__main__": + assert len(sys.argv) == 2, "Ohh no, audio file seems to be missing" + + audio_folder_path = sys.argv[1] + + # Initialize the Decoder + obj = Decoder('large-v3', language='mr') + + # Create a DataFrame to store file names and corresponding transcripts + transcripts_df = pd.DataFrame(columns=['MP3_File', 'Transcript']) + count=0 + + # Iterate through all MP3 files in the folder + for filename in os.listdir(audio_folder_path): + if filename.endswith(".mp3"): + mp3_file_path = os.path.join(audio_folder_path, filename) + + # Decode the MP3 file + asr_output = obj.decode(mp3_file_path) + + # print(asr_output) + + # Append the file name and transcript to the DataFrame + transcripts_df = transcripts_df.append({'MP3_File': filename, 'Transcript': asr_output}, ignore_index=True) + count+=1 + if count % 10 == 0: + print(f'{count} files done') + + # Save the transcript to a text file + output_dir = "./" + # asr_save_path = os.path.join(output_dir, filename.replace(".mp3", ".txt")) + # with open(asr_save_path, 'w') as f: + # f.write(asr_output) + + # Save the DataFrame to a CSV file + csv_save_path = os.path.join(output_dir, "transcripts_marathi.csv") + transcripts_df.to_csv(csv_save_path, index=False) + + print("Transcription and CSV file creation completed.") diff --git a/ST/inference/uploaded_audio.wav b/ST/inference/uploaded_audio.wav new file mode 100644 index 0000000000000000000000000000000000000000..5c45b5ada08809c7b4b2c6ebeef0adf1c17a340f Binary files /dev/null and b/ST/inference/uploaded_audio.wav differ