Fixup dependencies for package test (#1791)

* Fixup dependencies for package test

* Explicitly depend on protobuf

* Add changelog

* Fix test dependencies

* Install proper protobuf dependency

* Add stubs for v5.27.4 and v5.28.3, required due to incompatibilities with v5.26.1 and higher minor vers

* Change to generate with minimum supported grpcio-tools dep for pb major ver

* Add test to matrix of incompat grpcio with pb

* Raise custom exception on incompatible grpc/pb versions for stubs

* Add test verifying compatability throw for all grpcio/protobuf combinations

* Add install of test deps in new test

* Remove env var drilling in test

* Move test to standalone package to avoid imports outside of test

* Add missed init for pkg

* Move weaviate import into test

* Except the expected connection error in new test

* Fix test for wrong protos

* Remove other import

* Fix checking of exception message

* More fixes

* Remove yanked dep in test

---------

Co-authored-by: Tommy Smith <[email protected]>

Author: dirkkul

.github/workflows/main.yaml

@@ -97,6 +97,26 @@ jobs:
           name: coverage-report-${{ matrix.folder }}
           path: coverage-${{ matrix.folder }}.xml
 
+  proto-test:
+    name: Run importing protos test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        grpc: ["1.59.5", "1.63.0", "1.65.0", "1.66.0", "1.68.0", "1.72.1", "1.73.0", "1.74.0"]
+        protobuf: ["4.25.8", "5.26.0", "5.27.4", "5.28.3", "5.29.0", "6.30.0", "6.31.1", "6.32.0"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: 'pip' # caching pip dependencies
+      - run: |
+          pip install -r requirements-test.txt  -r requirements-devel.txt
+          pip install grpcio==${{ matrix.grpc }} protobuf==${{ matrix.protobuf }}
+      - name: Run proto test
+        run: pytest proto_test/test_proto.py
+
   integration-tests-embedded:
     name: Run Integration Tests Embedded
     runs-on: ubuntu-latest
@@ -133,11 +153,11 @@ jobs:
       fail-fast: false
       matrix:
         versions: [
-          { py: "3.9", weaviate: $WEAVIATE_131, grpc-version: "1.59.0"},
-          { py: "3.10", weaviate: $WEAVIATE_131, grpc-version: "1.66.0"},
-          { py: "3.11", weaviate: $WEAVIATE_131, grpc-version: "1.70.0"},
-          { py: "3.12", weaviate: $WEAVIATE_131, grpc-version: "1.70.0"},
-          { py: "3.13", weaviate: $WEAVIATE_131, grpc-version: "1.70.0"}
+          { py: "3.9", weaviate: $WEAVIATE_131, grpc: "1.59.0"},
+          { py: "3.10", weaviate: $WEAVIATE_131, grpc: "1.66.0"},
+          { py: "3.11", weaviate: $WEAVIATE_131, grpc: "1.70.0"},
+          { py: "3.12", weaviate: $WEAVIATE_131, grpc: "1.72.1"},
+          { py: "3.13", weaviate: $WEAVIATE_131, grpc: "1.74.0"}
         ]
         optional_dependencies: [false]
     steps:
@@ -157,7 +177,7 @@ jobs:
           password: ${{secrets.DOCKER_PASSWORD}}
       - run: |
           pip install -r requirements-test.txt
-          pip install grpcio==${{ matrix.versions.grpc-version }} grpcio-tools==${{ matrix.versions.grpc-version }} grpcio-health-checking==${{ matrix.versions.grpc-version }}
+          pip install grpcio==${{ matrix.versions.grpc }} grpcio-tools==${{ matrix.versions.grpc }} grpcio-health-checking==${{ matrix.versions.grpc }}
           pip install .
       - name: start weaviate
         run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
@@ -278,14 +298,14 @@ jobs:
       fail-fast: false
       matrix:
         version: [
-          { extra: "1.62.0", server: $WEAVIATE_125},
-          { extra: "1.66.2", server: $WEAVIATE_126},
-          { extra: "1.70.0", server: $WEAVIATE_127},
-          { extra: "1.59.5", server: $WEAVIATE_128},
-          { extra: "1.62.0", server: $WEAVIATE_129},
-          { extra: "1.69.0", server: $WEAVIATE_130},
-          { extra: "1.70.0", server: $WEAVIATE_131},
-          { extra: "1.71.0", server: $WEAVIATE_132}
+          $WEAVIATE_125,
+          $WEAVIATE_126,
+          $WEAVIATE_127,
+          $WEAVIATE_128,
+          $WEAVIATE_129,
+          $WEAVIATE_130,
+          $WEAVIATE_131,
+          $WEAVIATE_132
         ]
     steps:
       - name: Checkout
@@ -305,16 +325,15 @@ jobs:
       - run: |
           pip install -r requirements-test.txt  # install test dependencies
           pip install weaviate_client-*.whl
-          pip install grpcio==${{ matrix.version.extra }} grpcio-tools==${{ matrix.version.extra }} grpcio-health-checking==${{ matrix.version.extra }}
       - run: rm -r weaviate
       - name: start weaviate
-        run: /bin/bash ci/start_weaviate.sh ${{ matrix.version.server }}
+        run: /bin/bash ci/start_weaviate.sh ${{ matrix.version }}
       - name: Run integration tests without auth secrets (auth tests ran in previous job)
         run: pytest -v -n auto --dist loadgroup integration
 
   build-and-publish:
     name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
-    needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package]
+    needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

docs/changelog.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+Version 4.16.9
+--------------
+This patch version includes:
+    - Explicitly depend on protobuf package
+
 Version 4.16.8
 --------------
 This patch version includes:

proto_test/__init__.py

@@ -0,0 +1,19 @@
+import pytest
+from importlib.metadata import version as metadata_version
+from packaging import version
+
+
+def test_proto_import():
+    grpc_ver = version.parse(metadata_version("grpcio"))
+    pb_ver = version.parse(metadata_version("protobuf"))
+
+    if (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or (
+        pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0")
+    ):
+        with pytest.raises(Exception) as exc_info:
+            import weaviate
+        assert "gRPC incompatibility detected" in str(exc_info.value)
+    else:
+        import weaviate
+
+        assert weaviate.version is not None

proto_test/test_proto.py

@@ -39,6 +39,7 @@ install_requires =
     authlib>=1.2.1,<2.0.0
     pydantic>=2.8.0,<3.0.0
     grpcio>=1.59.5,<1.80.0
+    protobuf>=4.21.6,<7.0.0
     deprecation>=2.1.0,<3.0.0
 python_requires = >=3.9
 

setup.cfg

@@ -6,6 +6,7 @@
 import httpx
 from grpc import Call, StatusCode  # type: ignore
 from grpc.aio import AioRpcError  # type: ignore
+from packaging import version
 
 ERROR_CODE_EXPLANATION = {
     413: """Payload Too Large. Try to decrease the batch size or increase the maximum request size on your weaviate
@@ -375,3 +376,10 @@ def __init__(self) -> None:
         super().__init__(
             'Weaviate Agents (Alpha) functionality requires additional dependencies. Please install them using: "pip install weaviate-client[agents]"'
         )
+
+
+class WeaviateProtobufIncompatibility(Exception):
+    def __init__(self, pb: version.Version, grpc: version.Version) -> None:
+        super().__init__(
+            f"gRPC incompatibility detected. Protobuf: {pb.base_version}, gRPC: {grpc.base_version}. Ensure that your protobuf and grpcio versions are compatible or runtime errors may occur."
+        )

weaviate/exceptions.py

@@ -13,20 +13,24 @@
 
 from importlib.metadata import version as metadata_version
 
-def get_protobuf_version() -> version.Version:
-    """Get the installed protobuf version."""
-    return version.parse(metadata_version('protobuf'))
+from weaviate.exceptions import WeaviateProtobufIncompatibility
 
+def get_version(pkg: str)-> version.Version:
+    return version.parse(metadata_version(pkg))
 
-pb_version = get_protobuf_version()
+pb_version, grpc_version = get_version("protobuf"), get_version("grpcio")
 if pb_version >= version.parse("6.30.0"):
+    if grpc_version < version.parse("1.72.0"):
+        raise WeaviateProtobufIncompatibility(pb_version, grpc_version)
     from weaviate.proto.v1.v6300.v1 import weaviate_pb2_grpc, aggregate_pb2, base_pb2, base_search_pb2, batch_delete_pb2, batch_pb2, generative_pb2, health_pb2, health_pb2_grpc, properties_pb2, search_get_pb2, tenants_pb2
 elif pb_version >= version.parse("5.26.1"):
+    if grpc_version < version.parse("1.63.0"):
+        raise WeaviateProtobufIncompatibility(pb_version, grpc_version)
     from weaviate.proto.v1.v5261.v1 import weaviate_pb2_grpc, aggregate_pb2, base_pb2, base_search_pb2, batch_delete_pb2, batch_pb2, generative_pb2, health_pb2, health_pb2_grpc, properties_pb2, search_get_pb2, tenants_pb2
 elif pb_version >= version.parse("4.21.6"):
     from weaviate.proto.v1.v4216.v1 import weaviate_pb2_grpc, aggregate_pb2, base_pb2, base_search_pb2, batch_delete_pb2, batch_pb2, generative_pb2, health_pb2, health_pb2_grpc, properties_pb2, search_get_pb2, tenants_pb2
 else:
-    raise RuntimeError(f"Unsupported grpcio-tools version: {pb_version}. Only versions 1.50.0+ and <1.80 are supported.")
+    raise RuntimeError(f"Unsupported protobuf version: {pb_version}. Only versions 4.21.6+ are supported.")
 
 __all__ = [
     "aggregate_pb2", "base_pb2", "base_search_pb2", "batch_delete_pb2", "batch_pb2", "generative_pb2", "health_pb2", "health_pb2_grpc", "properties_pb2", "search_get_pb2", "tenants_pb2", "weaviate_pb2_grpc"