Migration numPy 2.0 and remove deprecated scipy sparsetools functions

.github/workflows/build-wheels.yml

@@ -36,15 +36,15 @@ jobs:
           platforms: all
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v3.1.4
         env:
           CIBW_ARCHS_LINUX: x86_64 aarch64
           CIBW_ARCHS_MACOS: x86_64 arm64
           CIBW_ARCHS_WINDOWS: AMD64 x86
-          CIBW_SKIP: pp* cp36-* cp37-* cp313-* *-win32 *_i686 *-musllinux_*
+          CIBW_SKIP: pp* cp38-* cp314* *-win32 *_i686 *-musllinux_*
           CIBW_TEST_COMMAND: pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim
           CIBW_TEST_REQUIRES: pytest testfixtures mock
-          CIBW_TEST_SKIP: cp38* cp39* cp310* cp311* cp313* *_aarch64 *_arm64 *_universal2
+          CIBW_TEST_SKIP: cp38* cp39* cp310* cp311* cp314* *_aarch64 *_arm64 *_universal2
           CIBW_BUILD_VERBOSITY: 3
 
       - name: Upload wheels as artifacts
@@ -61,21 +61,18 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {python: '3.8', os: macos-latest}
           - {python: '3.9', os: macos-latest}
           - {python: '3.10', os: macos-latest}
           - {python: '3.11', os: macos-latest}
           - {python: '3.12', os: macos-latest}
           - {python: '3.13', os: macos-latest}
 
-          - {python: '3.8', os: ubuntu-24.04}
           - {python: '3.9', os: ubuntu-24.04}
           - {python: '3.10', os: ubuntu-24.04}
           - {python: '3.11', os: ubuntu-24.04}
           - {python: '3.12', os: ubuntu-24.04}
           - {python: '3.13', os: ubuntu-24.04}
 
-          - {python: '3.8', os: windows-2025}
           - {python: '3.9', os: windows-2025}
           - {python: '3.10', os: windows-2025}
           - {python: '3.11', os: windows-2025}

.github/workflows/tests.yml

@@ -33,17 +33,17 @@ jobs:
           #
           # We use Py3.8 here for historical reasons.
           #
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Update pip
-        run: python -m pip install -U pip
+        run: python -m pip install -U pip setuptools
 
       - name: Install apt packages for LaTeX rendering
         run: |
           sudo apt-get -yq update
           sudo apt-get -yq remove texlive-binaries --purge
           sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk
-          sudo apt-get -yq install build-essential python3.8-dev
+          sudo apt-get -yq install build-essential
       - name: Install gensim and its dependencies
         run: pip install -e .[docs]
 
@@ -63,17 +63,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {python: '3.8', os: ubuntu-24.04}
           - {python: '3.9', os: ubuntu-24.04}
           - {python: '3.10', os: ubuntu-24.04}
           - {python: '3.11', os: ubuntu-24.04}
           - {python: '3.12', os: ubuntu-24.04}
+          - {python: '3.13', os: ubuntu-24.04}
 
-          - {python: '3.8', os: windows-2025}
           - {python: '3.9', os: windows-2025}
           - {python: '3.10', os: windows-2025}
           - {python: '3.11', os: windows-2025}
           - {python: '3.12', os: windows-2025}
+          - {python: '3.13', os: windows-2025}
 
     #
     # Don't run this job unless the linters have succeeded.

gensim/_matutils.pyx

@@ -31,6 +31,9 @@ def mean_absolute_difference(a, b):
     if a.shape != b.shape:
         raise ValueError("a and b must have same shape")
 
+    if a.size == 0:
+        return 0.0
+
     if a.dtype == np.float64:
         return _mean_absolute_difference[double](a, b)
     elif a.dtype == np.float32:
@@ -42,7 +45,7 @@ def mean_absolute_difference(a, b):
 @cython.boundscheck(False)
 @cython.wraparound(False)
 @cython.cdivision(True)
-cdef DTYPE_t _mean_absolute_difference(DTYPE_t[:] a, DTYPE_t[:] b) nogil:
+cdef DTYPE_t _mean_absolute_difference(DTYPE_t[:] a, DTYPE_t[:] b) noexcept nogil:
     """Mean absolute difference between two arrays.
 
     Parameters
@@ -86,12 +89,22 @@ def logsumexp(x):
     float
         log of sum of exponentials of elements in `x`.
 
+    Raises
+    ------
+    ValueError
+        If `x` is not a 2D array or if it is empty.
+
     Warnings
     --------
     By performance reasons, doesn't support NaNs or 1d, 3d, etc arrays like :func:`scipy.special.logsumexp`.
 
     """
 
+    if x.ndim != 2:
+        raise ValueError("Input must be a 2D array.")
+    if x.shape[0] == 0 or x.shape[1] == 0:
+        raise ValueError("Input must not be empty.")
+
     if x.dtype == np.float64:
         return _logsumexp_2d[double](x)
     elif x.dtype == np.float32:
@@ -103,7 +116,7 @@ def logsumexp(x):
 @cython.boundscheck(False)
 @cython.wraparound(False)
 @cython.cdivision(True)
-cdef DTYPE_t _logsumexp_2d(DTYPE_t[:, :] data) nogil:
+cdef DTYPE_t _logsumexp_2d(DTYPE_t[:, :] data) noexcept nogil:
     """Log of sum of exponentials.
 
     Parameters
@@ -156,6 +169,9 @@ def dirichlet_expectation(alpha):
         Log of expected values, dimension same as `alpha.ndim`.
 
     """
+    if not isinstance(alpha, np.ndarray):
+        raise ValueError("alpha must be a numpy.ndarray")
+
     if alpha.ndim == 2:
         return dirichlet_expectation_2d(alpha)
     else:
@@ -223,7 +239,7 @@ def dirichlet_expectation_1d(alpha):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-cdef void _dirichlet_expectation_1d(DTYPE_t[:] alpha, DTYPE_t[:] out) nogil:
+cdef void _dirichlet_expectation_1d(DTYPE_t[:] alpha, DTYPE_t[:] out) noexcept nogil:
     """Expected value of log(theta) where theta is drawn from a Dirichlet distribution.
 
     Parameters
@@ -251,7 +267,7 @@ cdef void _dirichlet_expectation_1d(DTYPE_t[:] alpha, DTYPE_t[:] out) nogil:
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-cdef void _dirichlet_expectation_2d(DTYPE_t[:, :] alpha, DTYPE_t[:, :] out) nogil:
+cdef void _dirichlet_expectation_2d(DTYPE_t[:, :] alpha, DTYPE_t[:, :] out) noexcept nogil:
     """Expected value of log(theta) where theta is drawn from a Dirichlet distribution.
 
     Parameters
@@ -298,7 +314,7 @@ def digamma(DTYPE_t x):
 
 
 @cython.cdivision(True)
-cdef inline DTYPE_t _digamma(DTYPE_t x,) nogil:
+cdef inline DTYPE_t _digamma(DTYPE_t x,) noexcept nogil:
     """Digamma function for positive floats.
 
     Parameters

gensim/models/doc2vec_corpusfile.pyx

@@ -61,7 +61,7 @@ cdef void prepare_c_structures_for_batch(
         np.uint32_t *indexes, int *codelens, np.uint8_t **codes, np.uint32_t **points,
         np.uint32_t *reduced_windows, int *document_len, int train_words,
         int docvecs_count, int doc_tag, int shrink_windows,
-    ) nogil:
+    ) noexcept nogil:
     cdef VocabItem predict_word
     cdef string token
     cdef int i = 0

gensim/models/doc2vec_inner.pxd

@@ -57,39 +57,39 @@ cdef void fast_document_dbow_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen,
     REAL_t *context_vectors, REAL_t *syn1, const int size,
     const np.uint32_t context_index, const REAL_t alpha, REAL_t *work, int learn_context, int learn_hidden,
-    REAL_t *contexts_lockf, const np.uint32_t contexts_lockf_len) nogil
+    REAL_t *contexts_lockf, const np.uint32_t contexts_lockf_len) noexcept nogil
 
 
 cdef unsigned long long fast_document_dbow_neg(
     const int negative, np.uint32_t *cum_table, unsigned long long cum_table_len,
     REAL_t *context_vectors, REAL_t *syn1neg, const int size, const np.uint32_t word_index,
     const np.uint32_t context_index, const REAL_t alpha, REAL_t *work,
     unsigned long long next_random, int learn_context, int learn_hidden, REAL_t *contexts_lockf,
-    const np.uint32_t contexts_lockf_len) nogil
+    const np.uint32_t contexts_lockf_len) noexcept nogil
 
 
 cdef void fast_document_dm_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len,
     REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work,
-    const int size, int learn_hidden) nogil
+    const int size, int learn_hidden) noexcept nogil
 
 
 cdef unsigned long long fast_document_dm_neg(
     const int negative, np.uint32_t *cum_table, unsigned long long cum_table_len, unsigned long long next_random,
     REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work,
-    const int size, int learn_hidden) nogil
+    const int size, int learn_hidden) noexcept nogil
 
 
 cdef void fast_document_dmc_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len,
     REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work,
-    const int layer1_size, const int vector_size, int learn_hidden) nogil
+    const int layer1_size, const int vector_size, int learn_hidden) noexcept nogil
 
 
 cdef unsigned long long fast_document_dmc_neg(
     const int negative, np.uint32_t *cum_table, unsigned long long cum_table_len, unsigned long long next_random,
     REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work,
-    const int layer1_size, const int vector_size, int learn_hidden) nogil
+    const int layer1_size, const int vector_size, int learn_hidden) noexcept nogil
 
 
 cdef init_d2v_config(Doc2VecConfig *c, model, alpha, learn_doctags, learn_words, learn_hidden, train_words=*, work=*,

gensim/models/doc2vec_inner.pyx

@@ -39,7 +39,7 @@ cdef void fast_document_dbow_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen,
     REAL_t *context_vectors, REAL_t *syn1, const int size,
     const np.uint32_t context_index, const REAL_t alpha, REAL_t *work, int learn_context, int learn_hidden,
-    REAL_t *contexts_lockf, const np.uint32_t contexts_lockf_len) nogil:
+    REAL_t *contexts_lockf, const np.uint32_t contexts_lockf_len) noexcept nogil:
 
     cdef long long a, b
     cdef long long row1 = context_index * size, row2
@@ -66,7 +66,7 @@ cdef unsigned long long fast_document_dbow_neg(
     REAL_t *context_vectors, REAL_t *syn1neg, const int size, const np.uint32_t word_index,
     const np.uint32_t context_index, const REAL_t alpha, REAL_t *work,
     unsigned long long next_random, int learn_context, int learn_hidden, REAL_t *contexts_lockf,
-    const np.uint32_t contexts_lockf_len) nogil:
+    const np.uint32_t contexts_lockf_len) noexcept nogil:
 
     cdef long long a
     cdef long long row1 = context_index * size, row2
@@ -106,7 +106,7 @@ cdef unsigned long long fast_document_dbow_neg(
 cdef void fast_document_dm_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len,
     REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work,
-    const int size, int learn_hidden) nogil:
+    const int size, int learn_hidden) noexcept nogil:
 
     cdef long long b
     cdef long long row2
@@ -129,7 +129,7 @@ cdef void fast_document_dm_hs(
 cdef unsigned long long fast_document_dm_neg(
     const int negative, np.uint32_t *cum_table, unsigned long long cum_table_len, unsigned long long next_random,
     REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work,
-    const int size, int learn_hidden) nogil:
+    const int size, int learn_hidden) noexcept nogil:
 
     cdef long long row2
     cdef unsigned long long modulo = 281474976710655ULL
@@ -165,7 +165,7 @@ cdef unsigned long long fast_document_dm_neg(
 cdef void fast_document_dmc_hs(
     const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len,
     REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work,
-    const int layer1_size, const int vector_size, int learn_hidden) nogil:
+    const int layer1_size, const int vector_size, int learn_hidden) noexcept nogil:
 
     cdef long long a, b
     cdef long long row2
@@ -189,7 +189,7 @@ cdef void fast_document_dmc_hs(
 cdef unsigned long long fast_document_dmc_neg(
     const int negative, np.uint32_t *cum_table, unsigned long long cum_table_len, unsigned long long next_random,
     REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work,
-    const int layer1_size, const int vector_size, int learn_hidden) nogil:
+    const int layer1_size, const int vector_size, int learn_hidden) noexcept nogil:
 
     cdef long long a
     cdef long long row2

gensim/models/fasttext_corpusfile.pyx

@@ -48,7 +48,7 @@ cdef void prepare_c_structures_for_batch(
         int *effective_words, int *effective_sentences, unsigned long long *next_random, cvocab_t *vocab,
         int *sentence_idx, np.uint32_t *indexes, int *codelens, np.uint8_t **codes, np.uint32_t **points,
         np.uint32_t *reduced_windows, int *subwords_idx_len, np.uint32_t **subwords_idx, int shrink_windows,
-    ) nogil:
+    ) noexcept nogil:
     cdef VocabItem word
     cdef string token
     cdef vector[string] sent

gensim/models/fasttext_inner.pxd

@@ -135,16 +135,16 @@ cdef void init_ft_config(FastTextConfig *c, model, alpha, _work, _neu1)
 cdef object populate_ft_config(FastTextConfig *c, vocab, buckets_word, sentences)
 
 
-cdef void fasttext_fast_sentence_sg_neg(FastTextConfig *c, int i, int j) nogil
+cdef void fasttext_fast_sentence_sg_neg(FastTextConfig *c, int i, int j) noexcept nogil
 
 
-cdef void fasttext_fast_sentence_sg_hs(FastTextConfig *c, int i, int j) nogil
+cdef void fasttext_fast_sentence_sg_hs(FastTextConfig *c, int i, int j) noexcept nogil
 
 
-cdef void fasttext_fast_sentence_cbow_neg(FastTextConfig *c, int i, int j, int k) nogil
+cdef void fasttext_fast_sentence_cbow_neg(FastTextConfig *c, int i, int j, int k) noexcept nogil
 
 
-cdef void fasttext_fast_sentence_cbow_hs(FastTextConfig *c, int i, int j, int k) nogil
+cdef void fasttext_fast_sentence_cbow_hs(FastTextConfig *c, int i, int j, int k) noexcept nogil
 
 
-cdef void fasttext_train_any(FastTextConfig *c, int num_sentences) nogil
+cdef void fasttext_train_any(FastTextConfig *c, int num_sentences) noexcept nogil

gensim/models/fasttext_inner.pyx

@@ -72,7 +72,7 @@ cdef int ONE = 1
 cdef REAL_t ONEF = <REAL_t>1.0
 
 
-cdef void fasttext_fast_sentence_sg_neg(FastTextConfig *c, int i, int j) nogil:
+cdef void fasttext_fast_sentence_sg_neg(FastTextConfig *c, int i, int j) noexcept nogil:
     """Perform skipgram training with negative sampling.
 
     Parameters
@@ -145,7 +145,7 @@ cdef void fasttext_fast_sentence_sg_neg(FastTextConfig *c, int i, int j) nogil:
                   c.work, &ONE, &c.syn0_ngrams[subwords_index[d]*c.size], &ONE)
 
 
-cdef void fasttext_fast_sentence_sg_hs(FastTextConfig *c, int i, int j) nogil:
+cdef void fasttext_fast_sentence_sg_hs(FastTextConfig *c, int i, int j) noexcept nogil:
     """Perform skipgram training with hierarchical sampling.
 
     Parameters
@@ -221,7 +221,7 @@ cdef void fasttext_fast_sentence_sg_hs(FastTextConfig *c, int i, int j) nogil:
             &c.syn0_ngrams[row2], &ONE)
 
 
-cdef void fasttext_fast_sentence_cbow_neg(FastTextConfig *c, int i, int j, int k) nogil:
+cdef void fasttext_fast_sentence_cbow_neg(FastTextConfig *c, int i, int j, int k) noexcept nogil:
     """Perform CBOW training with negative sampling.
 
     Parameters
@@ -306,7 +306,7 @@ cdef void fasttext_fast_sentence_cbow_neg(FastTextConfig *c, int i, int j, int k
                 &c.syn0_ngrams[c.subwords_idx[m][d]*c.size], &ONE)
 
 
-cdef void fasttext_fast_sentence_cbow_hs(FastTextConfig *c, int i, int j, int k) nogil:
+cdef void fasttext_fast_sentence_cbow_hs(FastTextConfig *c, int i, int j, int k) noexcept nogil:
     """Perform CBOW training with hierarchical sampling.
 
     Parameters
@@ -510,7 +510,7 @@ cdef object populate_ft_config(FastTextConfig *c, wv, buckets_word, sentences):
     return effective_words, effective_sentences
 
 
-cdef void fasttext_train_any(FastTextConfig *c, int num_sentences) nogil:
+cdef void fasttext_train_any(FastTextConfig *c, int num_sentences) noexcept nogil:
     """Performs training on a fully initialized and populated configuration.
 
     Parameters

gensim/models/keyedvectors.py

@@ -495,9 +495,9 @@ def get_mean_vector(self, keys, weights=None, pre_normalize=True, post_normalize
         if len(keys) == 0:
             raise ValueError("cannot compute mean with no input")
         if isinstance(weights, list):
-            weights = np.array(weights)
+            weights = np.array(weights, dtype=self.vectors.dtype)
         if weights is None:
-            weights = np.ones(len(keys))
+            weights = np.ones(len(keys), dtype=self.vectors.dtype)
         if len(keys) != weights.shape[0]:  # weights is a 1-D numpy array
             raise ValueError(
                 "keys and weights array must have same number of elements"
@@ -746,7 +746,9 @@ def sort_by_descending_frequency(self):
         """Sort the vocabulary so the most frequent words have the lowest indexes."""
         if not len(self):
             return  # noop if empty
-        count_sorted_indexes = np.argsort(self.expandos['count'])[::-1]
+        counts = np.asarray(self.expandos['count'], dtype=np.int64)
+        indices = np.arange(len(counts))
+        count_sorted_indexes = np.lexsort((-indices, -counts))
         self.index_to_key = [self.index_to_key[idx] for idx in count_sorted_indexes]
         self.allocate_vecattrs()
         for k in self.expandos:
@@ -1667,7 +1669,7 @@ def save_word2vec_format(
                 if binary:
                     fout.write(f"{prefix}{key} ".encode('utf8') + key_vector.astype(REAL).tobytes())
                 else:
-                    fout.write(f"{prefix}{key} {' '.join(repr(val) for val in key_vector)}\n".encode('utf8'))
+                    fout.write(f"{prefix}{key} {' '.join(str(val) for val in key_vector)}\n".encode('utf8'))
 
     @classmethod
     def load_word2vec_format(
@@ -1977,7 +1979,7 @@ def _word2vec_read_text(fin, kv, counts, vocab_size, vector_size, datatype, unic
 
 def _word2vec_line_to_vector(line, datatype, unicode_errors, encoding):
     parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(" ")
-    word, weights = parts[0], [datatype(x) for x in parts[1:]]
+    word, weights = parts[0], [datatype(x).item() for x in parts[1:]]
     return word, weights