[Dependencies] Remove tokenizers requirement. (#227)

Duyi-Wang · web-flow · commit 084dc20f0ec0 · 2024-02-20T15:03:17.000+08:00
diff --git a/examples/web_demo/requirements.txt b/examples/web_demo/requirements.txt
@@ -2,7 +2,6 @@
 gradio==4.11.0
 gradio_client==0.7.3
 sentencepiece==0.1.99
-tokenizers==0.13.3
 torch==2.0.1+cpu
 transformers==4.36.0
 transformers_stream_generator==0.0.4
diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,6 @@
 -f https://download.pytorch.org/whl/torch_stable.html
 cmake==3.26.1
 sentencepiece==0.1.99
-tokenizers==0.13.3
 torch==2.0.1+cpu
 transformers==4.36.0
 accelerate==0.23.0
