Upgrading dependencies.

Author: Narsil

bindings/python/Cargo.toml

@@ -14,17 +14,17 @@ serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.11"
-pyo3 = { version = "0.24.2", features = ["abi3", "abi3-py39", "py-clone"] }
-numpy = "0.24"
+pyo3 = { version = "0.25", features = ["abi3", "abi3-py39", "py-clone"] }
+numpy = "0.25"
 ndarray = "0.16"
-itertools = "0.12"
+itertools = "0.14"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.10"
-pyo3 = { version = "0.24.2", features = ["auto-initialize"] }
+pyo3 = { version = "0.25", features = ["auto-initialize"] }
 
 [features]
 default = ["pyo3/extension-module"]

tokenizers/Cargo.toml

@@ -42,19 +42,19 @@ required-features = ["http"]
 harness = false
 
 [dependencies]
-rand = "0.8"
+rand = "0.9"
 onig = { version = "6.5.1", default-features = false, optional = true }
 regex = "1.10"
 regex-syntax = "0.8"
 rayon = "1.10"
-rayon-cond = "0.3"
+rayon-cond = "0.4"
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"
 unicode-normalization-alignments = "0.1"
 unicode_categories = "0.1"
 unicode-segmentation = "1.11"
 indicatif = {version = "0.17", optional = true}
-itertools = "0.13"
+itertools = "0.14"
 log = "0.4"
 derive_builder = "0.20"
 spm_precompiled = "0.1.3"
@@ -64,7 +64,7 @@ paste = "1.0.14"
 macro_rules_attribute = "0.2.0"
 thiserror = "2"
 fancy-regex = { version = "0.14", optional = true}
-getrandom = { version = "0.2.10" }
+getrandom = { version = "0.3" }
 esaxx-rs = { version = "0.1.10", default-features = false, features=[]}
 monostate = "0.1.12"
 
@@ -73,11 +73,11 @@ default = ["progressbar", "onig", "esaxx_fast"]
 esaxx_fast = ["esaxx-rs/cpp"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]
-unstable_wasm = ["fancy-regex", "getrandom/js"]
+unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
 rustls-tls = ["hf-hub?/rustls-tls"]
 
 [dev-dependencies]
-criterion = "0.5"
+criterion = "0.6"
 tempfile = "3.10"
 assert_approx_eq = "1.1"
 tracing = "0.1"

tokenizers/benches/common/mod.rs

@@ -1,6 +1,6 @@
 use std::time::{Duration, Instant};
 
-use criterion::black_box;
+use std::hint::black_box;
 
 use tokenizers::{
     Decoder, EncodeInput, Model, Normalizer, PostProcessor, PreTokenizer, TokenizerImpl, Trainer,

tokenizers/benches/layout_benchmark.rs

@@ -6,8 +6,8 @@ use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::time::{Duration, Instant};
 
-use criterion::black_box;
 use criterion::Criterion;
+use std::hint::black_box;
 use tokenizers::processors::template::TemplateProcessing;
 use tokenizers::{EncodeInput, Encoding, PostProcessor, Tokenizer};
 

tokenizers/benches/llama3.rs

@@ -1,6 +1,8 @@
 #[macro_use]
 extern crate criterion;
 
+use std::hint::black_box;
+
 use criterion::{Criterion, Throughput};
 use tokenizers::Tokenizer;
 
@@ -15,7 +17,7 @@ pub fn llama3(c: &mut Criterion) {
         let add_special_tokens = false;
         b.iter(|| {
             tokenizer
-                .encode_batch_char_offsets(criterion::black_box(data.clone()), add_special_tokens)
+                .encode_batch_char_offsets(black_box(data.clone()), add_special_tokens)
                 .unwrap()
         })
     });
@@ -26,7 +28,7 @@ pub fn llama3(c: &mut Criterion) {
         let add_special_tokens = false;
         b.iter(|| {
             tokenizer
-                .encode_batch(criterion::black_box(data.clone()), add_special_tokens)
+                .encode_batch(black_box(data.clone()), add_special_tokens)
                 .unwrap()
         })
     });

tokenizers/src/models/bpe/word.rs

@@ -1,5 +1,5 @@
 use super::Pair;
-use rand::{thread_rng, Rng};
+use rand::{rng, Rng};
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
 
@@ -177,10 +177,7 @@ impl Word {
         );
 
         while let Some(top) = queue.pop() {
-            if dropout
-                .map(|d| thread_rng().gen::<f32>() < d)
-                .unwrap_or(false)
-            {
+            if dropout.map(|d| rng().random::<f32>() < d).unwrap_or(false) {
                 skip.push(top);
             } else {
                 // Re-insert the skipped elements

tokenizers/src/models/unigram/lattice.rs

@@ -1,5 +1,5 @@
-use rand::distributions::WeightedIndex;
-use rand::prelude::*;
+use rand::distr::weighted::WeightedIndex;
+use rand::{prelude::*, rng};
 use std::cell::RefCell;
 use std::cmp::{min, Ordering};
 use std::collections::BinaryHeap;
@@ -397,7 +397,7 @@ impl<'a> Lattice<'a> {
             }
         }
 
-        let mut rng = thread_rng();
+        let mut rng = rng();
         let mut results: Vec<NodeRef> = vec![];
         let mut probs: Vec<f64> = vec![];
         let mut z = alpha[self.eos_node().borrow().node_id];