Merge pull request #1136 from crusaderky/pytest-run-parallel

Free-threading tests

Author: taku910

.github/workflows/wheel.yml

@@ -138,6 +138,36 @@ jobs:
         if: runner.os == 'Windows'
         run: echo "digests=$(sha256sum python/wheelhouse/* | base64 -w0)" >> $GITHUB_OUTPUT
 
+  free-threading:
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          sparse-checkout: |
+            python/test
+            data/botchan.txt
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13t"
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          path: python/wheelhouse
+          merge-multiple: true
+
+      - name: Install sentencepiece wheel
+        run: pip install --find-links=python/wheelhouse sentencepiece
+
+      - name: Install test dependencies
+        run: pip install pytest pytest-run-parallel
+      
+      - name: Run free-threading tests
+        working-directory: python
+        run: pytest -v --parallel-threads 4
+        
   gather-digests:
     needs: [build_wheels]
     outputs:

python/.gitignore

@@ -1,6 +1,6 @@
 /*.so
 /build
 /*.pickle
-/m_*
-/m.*
+/m*.model
+/m*.vocab
 /src/sentencepiece/package_data

python/test/sentencepiece_test.py

@@ -779,12 +779,13 @@ def test_batch(self):
     self.assertEqual(e1, e3)
 
   def test_pickle(self):
-    with open('sp.pickle', 'wb') as f:
+    tid = threading.get_native_id()
+    with open(f'sp_{tid}.pickle', 'wb') as f:
       pickle.dump(self.sp_, f)
 
     id1 = self.sp_.encode('hello world.', out_type=int)
 
-    with open('sp.pickle', 'rb') as f:
+    with open(f'sp_{tid}.pickle', 'rb') as f:
       sp = pickle.load(f)
 
     id2 = sp.encode('hello world.', out_type=int)