Metadata validation

.circleci/config.yml

@@ -81,6 +81,7 @@ jobs:
             - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics
             - run: isort --check-only tests src benchmarks datasets metrics
             - run: flake8 tests src benchmarks datasets metrics
+            - run: ./scripts/datasets_metadata_validator.py
 
     build_doc:
         working_directory: ~/datasets

datasets/spanish_billion_words/README.md

@@ -4,7 +4,7 @@ annotations_creators:
 language_creators:
 - expert-generated
 languages:
-- es
+- esXXX_CI_SHOULD_FAIL_HERE
 licenses:
 - cc-by-sa-4.0
 multilinguality:

scripts/datasets_metadata_validator.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+""" This script will run in CI and make sure all new changes to datasets readme files have valid metadata yaml headers.
+
+"""
+
+from pathlib import Path
+from subprocess import check_output
+from typing import List
+
+from pydantic import ValidationError
+
+from datasets.utils.metadata import DatasetMetadata
+
+
+def get_changed_files(repo_path: Path) -> List[Path]:
+    diff_output = check_output(["git", "diff", "--name-only", "HEAD..origin/master"], cwd=repo_path)
+    changed_files = [Path(repo_path, f) for f in diff_output.decode().splitlines()]
+    return changed_files
+
+
+if __name__ == "__main__":
+    import logging
+    from argparse import ArgumentParser
+
+    logging.basicConfig(level=logging.DEBUG)
+
+    ap = ArgumentParser()
+    ap.add_argument("--repo_path", type=Path, default=Path.cwd())
+    ap.add_argument("--check_all", action="store_true")
+    args = ap.parse_args()
+
+    repo_path: Path = args.repo_path
+    if args.check_all:
+        readmes = [dd / "README.md" for dd in (repo_path / "datasets").iterdir()]
+    else:
+        changed_files = get_changed_files(repo_path)
+        readmes = [
+            f
+            for f in changed_files
+            if f.exists() and f.name.lower() == "readme.md" and f.parent.parent.name == "datasets"
+        ]
+
+    failed: List[Path] = []
+    for readme in sorted(readmes):
+        try:
+            DatasetMetadata.from_readme(readme)
+            logging.debug(f"✅️ Validated '{readme.relative_to(repo_path)}'")
+        except ValidationError as e:
+            failed.append(readme)
+            logging.warning(f"❌ Failed to validate '{readme.relative_to(repo_path)}':\n{e}")
+        except Exception as e:
+            failed.append(readme)
+            logging.warning(f"⁉️ Something unexpected happened on '{readme.relative_to(repo_path)}':\n{e}")
+
+    if len(failed) > 0:
+        logging.info(f"❌ Failed on {len(failed)} files.")
+        exit(1)
+    else:
+        logging.info("All is well, keep up the good work 🤗!")
+        exit(0)

setup.py

@@ -56,8 +56,8 @@
 import os
 import sys
 
-from setuptools import find_packages
-from setuptools import setup
+from setuptools import find_packages, setup
+
 
 DOCLINES = __doc__.split("\n")
 
@@ -140,26 +140,33 @@
     "texttable>=1.6.3",
     "s3fs>=0.4.2",
     "Werkzeug>=1.0.1",
+    # metadata validation
+    "langcodes[data]>=3.1.0",
+    "pydantic>=1.8.1",
 ]
 
 if os.name == "nt":  # windows
     TESTS_REQUIRE.remove("faiss-cpu")  # faiss doesn't exist on windows
 else:
     # dependencies of unbabel-comet
     # only test if not on windows since there're issues installing fairseq on windows
-    TESTS_REQUIRE.extend([
-        "wget>=3.2",
-        "pytorch-nlp==0.5.0",
-        "pytorch_lightning",
-        "fastBPE==0.1.0",
-        "fairseq",
-    ])
+    TESTS_REQUIRE.extend(
+        [
+            "wget>=3.2",
+            "pytorch-nlp==0.5.0",
+            "pytorch_lightning",
+            "fastBPE==0.1.0",
+            "fairseq",
+        ]
+    )
 
 
 QUALITY_REQUIRE = [
     "black",
-    "isort",
     "flake8==3.7.9",
+    "isort",
+    "langcodes[data]>=3.1.0",
+    "pydantic>=1.8.1",
 ]
 
 

src/datasets/utils/metadata.py

@@ -0,0 +1,155 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+import langcodes as lc
+import yaml
+from pydantic import BaseModel, conlist, validator
+
+
+BASE_REF_URL = "https://github.com/huggingface/datasets/tree/master/src/datasets/utils"
+this_url = f"{BASE_REF_URL}/{__file__}"
+logger = logging.getLogger(__name__)
+
+
+def dict_from_readme(f: Path) -> Optional[Dict[str, List[str]]]:
+    with f.open() as fi:
+        content = [line.strip() for line in fi]
+
+    if content[0] == "---" and "---" in content[1:]:
+        yamlblock = "\n".join(content[1 : content[1:].index("---") + 1])
+        metada_dict = yaml.safe_load(yamlblock) or dict()
+        return metada_dict
+
+
+def load_json_resource(resource: str) -> Tuple[Dict, str]:
+    utils_dir = Path(__file__).parent
+    with open(utils_dir / "resources" / resource) as fi:
+        return json.load(fi), f"{BASE_REF_URL}/resources/{resource}"
+
+
+known_licenses, known_licenses_url = load_json_resource("licenses.json")
+known_task_ids, known_task_ids_url = load_json_resource("tasks.json")
+known_creators, known_creators_url = load_json_resource("creators.json")
+known_size_categories = ["unknown", "n<1K", "1K<n<10K", "10K<n<100K", "100K<n<1M", "n>1M"]
+known_multilingualities = {
+    "monolingual": "contains a single language",
+    "multilingual": "contains multiple languages",
+    "translation": "contains translated or aligned text",
+    "other": "other type of language distribution",
+}
+
+
+def tagset_validator(values: List[str], reference_values: List[str], name: str, url: str) -> List[str]:
+    for v in values:
+        if v not in reference_values:
+            raise ValueError(f"'{v}' is not a registered tag for '{name}', reference at {url}")
+    return values
+
+
+def splitter(values: List[Any], predicate_fn: Callable[[Any], bool]) -> Tuple[List[Any], List[Any]]:
+    trues, falses = list(), list()
+    for v in values:
+        if predicate_fn(v):
+            trues.append(v)
+        else:
+            falses.append(v)
+    return trues, falses
+
+
+class DatasetMetadata(BaseModel):
+    annotations_creators: conlist(str, min_items=1)
+    language_creators: conlist(str, min_items=1)
+    languages: conlist(str, min_items=1)
+    licenses: conlist(str, min_items=1)
+    multilinguality: conlist(str, min_items=1)
+    size_categories: conlist(str, min_items=1)
+    source_datasets: conlist(str, min_items=1)
+    task_categories: conlist(str, min_items=1)
+    task_ids: conlist(str, min_items=1)
+
+    @classmethod
+    def from_readme(cls, f: Path) -> "DatasetMetadata":
+        metadata_dict = dict_from_readme(f)
+        if metadata_dict is not None:
+            return cls(**metadata_dict)
+        else:
+            raise ValueError(f"did not find a yaml block in '{f}'")
+
+    @classmethod
+    def from_yaml_string(cls, string: str) -> "DatasetMetadata":
+        metada_dict = yaml.safe_load(string) or dict()
+        return cls(**metada_dict)
+
+    @classmethod
+    def empty(cls) -> "DatasetMetadata":
+        return cls(
+            annotations_creators=list(),
+            language_creators=list(),
+            languages=list(),
+            licenses=list(),
+            multilinguality=list(),
+            size_categories=list(),
+            source_datasets=list(),
+            task_categories=list(),
+            task_ids=list(),
+        )
+
+    @validator("annotations_creators")
+    def annotations_creators_must_be_in_known_set(cls, annotations_creators: List[str]) -> List[str]:
+        return tagset_validator(annotations_creators, known_creators["annotations"], "annotations", known_creators_url)
+
+    @validator("language_creators")
+    def language_creators_must_be_in_known_set(cls, language_creators: List[str]) -> List[str]:
+        return tagset_validator(language_creators, known_creators["language"], "annotations", known_creators_url)
+
+    @validator("languages")
+    def language_code_must_be_recognized(cls, languages: List[str]):
+        for code in languages:
+            try:
+                lc.get(code)
+            except lc.tag_parser.LanguageTagError:
+                raise ValueError(
+                    f"'{code}' is not recognised as a valid language code (BCP47 norm), you can refer to https://github.com/LuminosoInsight/langcodes"
+                )
+        return languages
+
+    @validator("licenses")
+    def licenses_must_be_in_known_set(cls, licenses: List[str]):
+        return tagset_validator(licenses, list(known_licenses.keys()), "licenses", known_licenses_url)
+
+    @validator("task_categories")
+    def task_category_must_be_in_known_set(cls, task_categories: List[str]):
+        # TODO: we're currently ignoring all values starting with 'other' as our task taxonomy is bound to change
+        #   in the near future and we don't want to waste energy in tagging against a moving taxonomy.
+        known_set = list(known_task_ids.keys())
+        others, to_validate = splitter(task_categories, lambda e: e.startswith("other"))
+        return [*tagset_validator(to_validate, known_set, "tasks_ids", known_task_ids_url), *others]
+
+    @validator("task_ids")
+    def task_id_must_be_in_known_set(cls, task_ids: List[str]):
+        # TODO: we're currently ignoring all values starting with 'other' as our task taxonomy is bound to change
+        #   in the near future and we don't want to waste energy in tagging against a moving taxonomy.
+        known_set = [tid for _cat, d in known_task_ids.items() for tid in d["options"]]
+        others, to_validate = splitter(task_ids, lambda e: e.startswith("other"))
+        return [*tagset_validator(to_validate, known_set, "tasks_ids", known_task_ids_url), *others]
+
+    @validator("multilinguality")
+    def multilinguality_must_be_in_known_set(cls, multilinguality: List[str]):
+        return tagset_validator(multilinguality, list(known_multilingualities.keys()), "multilinguality", this_url)
+
+    @validator("size_categories")
+    def size_categories_must_be_in_known_set(cls, size_cats: List[str]):
+        return tagset_validator(size_cats, known_size_categories, "size_categories", this_url)
+
+
+if __name__ == "__main__":
+    from argparse import ArgumentParser
+
+    ap = ArgumentParser(usage="Validate the yaml metadata block of a README.md file.")
+    ap.add_argument("readme_filepath")
+    args = ap.parse_args()
+
+    readme_filepath = Path(args.readme_filepath)
+    DatasetMetadata.from_readme(readme_filepath)

src/datasets/utils/resources/creators.json

@@ -0,0 +1,17 @@
+{
+  "language": [
+    "found",
+    "crowdsourced",
+    "expert-generated",
+    "machine-generated",
+    "other"
+  ],
+  "annotations": [
+    "found",
+    "crowdsourced",
+    "expert-generated",
+    "machine-generated",
+    "no-annotation",
+    "other"
+  ]
+}