Install dependencies with uv in CI (#6779)

* Use uv to install dependencies

* Use blank pspacy models

* Nit

* Pin protobuf

Author: mariosasko

.github/workflows/ci.yml

@@ -53,18 +53,18 @@ jobs:
       - name: Pin setuptools-scm
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: echo "installing pinned version of setuptools-scm to fix seqeval installation on 3.7" && pip install "setuptools-scm==6.4.2"
+      - name: Install uv
+        run: pip install --upgrade uv
       - name: Install dependencies
         run: |
-          pip install .[tests,metrics-tests]
-          pip install -r additional-tests-requirements.txt --no-deps
-          python -m spacy download en_core_web_sm
-          python -m spacy download fr_core_news_sm
+          uv pip install --system "datasets[tests,metrics-tests] @ ."
+          uv pip install --system -r additional-tests-requirements.txt --no-deps
       - name: Install dependencies (latest versions)
         if: ${{ matrix.deps_versions == 'deps-latest' }}
-        run: pip install --upgrade pyarrow huggingface-hub dill
+        run: uv pip install --system --upgrade pyarrow huggingface-hub dill
       - name: Install dependencies (minimum versions)
         if: ${{ matrix.deps_versions != 'deps-latest' }}
-        run: pip install pyarrow==12.0.0 huggingface-hub==0.21.2 transformers dill==0.3.1.1
+        run: uv pip install --system pyarrow==12.0.0 huggingface-hub==0.21.2 transformers dill==0.3.1.1
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
@@ -88,8 +88,10 @@ jobs:
           python-version: "3.10"
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
       - name: Install dependencies
-        run: pip install .[tests]
+        run: uv pip install --system "datasets[tests] @ ."
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

setup.py

@@ -177,8 +177,8 @@
     "rarfile>=4.0",
     "sqlalchemy",
     "s3fs>=2021.11.1",  # aligned with fsspec[http]>=2021.11.1; test only on python 3.7 for now
-    "tensorflow>=2.3,!=2.6.0,!=2.6.1; sys_platform != 'darwin' or platform_machine != 'arm64'",
-    "tensorflow-macos; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "protobuf<4.0.0",  # 4.0.0 breaks compatibility with tensorflow<2.12
+    "tensorflow>=2.6.0",
     "tiktoken",
     "torch>=2.0.0",
     "soundfile>=0.12.1",
@@ -227,19 +227,17 @@
     # Following dependencies are required for the Python reference to be built properly
     "transformers",
     "torch",
-    "tensorflow>=2.2.0,!=2.6.0,!=2.6.1; sys_platform != 'darwin' or platform_machine != 'arm64'",
-    "tensorflow-macos; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "tensorflow>=2.6.0",
 ]
 
 EXTRAS_REQUIRE = {
     "audio": AUDIO_REQUIRE,
     "vision": VISION_REQUIRE,
     "apache-beam": ["apache-beam>=2.26.0"],
     "tensorflow": [
-        "tensorflow>=2.2.0,!=2.6.0,!=2.6.1; sys_platform != 'darwin' or platform_machine != 'arm64'",
-        "tensorflow-macos; sys_platform == 'darwin' and platform_machine == 'arm64'",
+        "tensorflow>=2.6.0",
     ],
-    "tensorflow_gpu": ["tensorflow-gpu>=2.2.0,!=2.6.0,!=2.6.1"],
+    "tensorflow_gpu": ["tensorflow>=2.6.0"],
     "torch": ["torch"],
     "jax": ["jax>=0.3.14", "jaxlib>=0.3.14"],
     "s3": ["s3fs"],

tests/test_fingerprint.py

@@ -23,7 +23,6 @@
     require_not_windows,
     require_regex,
     require_spacy,
-    require_spacy_model,
     require_tiktoken,
     require_torch,
     require_transformers,
@@ -331,17 +330,15 @@ def test_hash_torch_generator(self):
         self.assertNotEqual(hash1, hash2)
 
     @require_spacy
-    @require_spacy_model("en_core_web_sm")
-    @require_spacy_model("fr_core_news_sm")
     @pytest.mark.integration
     def test_hash_spacy_model(self):
         import spacy
 
-        nlp = spacy.load("en_core_web_sm")
+        nlp = spacy.blank("en")
         hash1 = Hasher.hash(nlp)
-        nlp = spacy.load("fr_core_news_sm")
+        nlp = spacy.blank("fr")
         hash2 = Hasher.hash(nlp)
-        nlp = spacy.load("en_core_web_sm")
+        nlp = spacy.blank("en")
         hash3 = Hasher.hash(nlp)
         self.assertEqual(hash1, hash3)
         self.assertNotEqual(hash1, hash2)

tests/utils.py

@@ -234,28 +234,6 @@ def require_spacy(test_case):
         return test_case
 
 
-def require_spacy_model(model):
-    """
-    Decorator marking a test that requires a spacy model.
-
-    These tests are skipped when they aren't installed.
-    """
-
-    def _require_spacy_model(test_case):
-        try:
-            import spacy  # noqa F401
-
-            spacy.load(model)
-        except ImportError:
-            return unittest.skip("test requires spacy")(test_case)
-        except OSError:
-            return unittest.skip("test requires spacy model '{}'".format(model))(test_case)
-        else:
-            return test_case
-
-    return _require_spacy_model
-
-
 def require_pyspark(test_case):
     """
     Decorator marking a test that requires pyspark.