Remove Setting.

Author: cryscan

assets/configs/Config.toml

@@ -9,12 +9,10 @@ max_runtime_batch = 8                                           # The maximum ba
 max_batch = 16                                                  # The maximum batches that are cached on GPU.
 embed_layer = 2                                                 # The (reversed) layer number whose output is used as embedding.
 embed_device = "Cpu"                                            # Device to put the embed tensor ("Cpu" or "Gpu").
+stop = ["\n\n"]                                                 # Additional stop words in generation.
 
 [tokenizer]
 path = "assets/tokenizer/rwkv_vocab_v20230424.json" # Path to the tokenizer.
 
 [adapter]
 Auto = {}
-
-[setting]
-stop = ["\n\n"] # Additional stop words in generation.

src/config.rs

@@ -12,7 +12,6 @@ pub struct Config {
     pub lora: Vec<Lora>,
     pub tokenizer: Tokenizer,
     pub adapter: AdapterOption,
-    pub setting: Setting,
 }
 
 impl From<Config> for ReloadRequest {
@@ -129,10 +128,3 @@ pub enum AdapterOption {
     Economical,
     Manual(usize),
 }
-
-/// More inference configurations.
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct Setting {
-    /// Additional stop words.
-    pub stop: Vec<String>,
-}

src/main.rs

@@ -14,7 +14,7 @@ use axum::{
     Router,
 };
 use clap::Parser;
-use config::{AdapterOption, Config, Setting};
+use config::{AdapterOption, Config};
 use flume::{Receiver, Sender};
 use itertools::Itertools;
 use memmap2::Mmap;
@@ -324,14 +324,14 @@ fn load_config(path: impl AsRef<Path>) -> Result<Config> {
 }
 
 #[tokio::main]
-async fn model_route(receiver: Receiver<ThreadRequest>, setting: Setting) -> Result<()> {
+async fn model_route(receiver: Receiver<ThreadRequest>) -> Result<()> {
     let env: Arc<RwLock<Environment>> = Default::default();
     let queue: Arc<Mutex<Vec<GenerateContext>>> = Default::default();
 
     let sender = {
         let (sender, receiver) = flume::unbounded();
         let env = env.clone();
-        tokio::task::spawn_blocking(move || run::run(receiver, env, setting));
+        tokio::task::spawn_blocking(move || run::run(receiver, env));
         sender
     };
 
@@ -575,20 +575,16 @@ async fn main() {
     let args = Args::parse();
     let (sender, receiver) = flume::unbounded::<ThreadRequest>();
 
-    let (setting, request) = {
+    let request = {
         let path = args
             .config
             .clone()
             .unwrap_or("assets/configs/Config.toml".into());
         log::info!("reading config {}...", path.to_string_lossy());
-        let config = load_config(path).expect("load config failed");
-
-        let setting = config.setting.clone();
-        let request = config.into();
-        (setting, request)
+        load_config(path).expect("load config failed").into()
     };
 
-    tokio::task::spawn_blocking(move || model_route(receiver, setting));
+    tokio::task::spawn_blocking(move || model_route(receiver));
     let _ = sender.send(ThreadRequest::Reload {
         request,
         sender: None,

src/run.rs

@@ -23,10 +23,7 @@ use web_rwkv::{
     tokenizer::Tokenizer,
 };
 
-use crate::{
-    config::Setting, Environment, FinishReason, GenerateRequest, Token, TokenCounter,
-    STATE_CHUNK_SIZE,
-};
+use crate::{Environment, FinishReason, GenerateRequest, Token, TokenCounter, STATE_CHUNK_SIZE};
 
 const PENALTY_FREE_LIST: [&str; 5] = ["\n", ",", ".", "\u{002c}", "\u{002f}"];
 
@@ -343,7 +340,6 @@ where
                     let context = self.model.context();
                     let info = self.model.info();
                     StateBuilder::new(context, info)
-                        .with_num_batch(1)
                         .with_chunk_size(STATE_CHUNK_SIZE)
                         .build_backed()
                 });
@@ -430,7 +426,7 @@ where
         }
     }
 
-    pub async fn process(&self, payloads: &mut [Payload], setting: &Setting) -> Result<()> {
+    pub async fn process(&self, payloads: &mut [Payload]) -> Result<()> {
         {
             let mut slots = self.slots.lock().await;
             let mut cache = self.backed.lock().await;
@@ -636,7 +632,6 @@ where
                 .request
                 .stop
                 .iter()
-                .chain(setting.stop.iter())
                 .map(|stop| {
                     let stop = stop.as_bytes();
                     let mut ptr_safe = 0;
@@ -729,9 +724,9 @@ macro_rules! impl_runtime_untyped {
             }
 
             #[inline]
-            pub async fn process(&self, payloads: &mut [Payload], setting: &Setting) -> Result<()> {
+            pub async fn process(&self, payloads: &mut [Payload]) -> Result<()> {
                 match self {
-                    $(RuntimeUntyped::$variant(runtime) => runtime.process(payloads, setting).await,)*
+                    $(RuntimeUntyped::$variant(runtime) => runtime.process(payloads).await,)*
                 }
             }
         }
@@ -741,12 +736,12 @@ macro_rules! impl_runtime_untyped {
 impl_runtime_untyped!(V4, V5, V6);
 
 #[tokio::main]
-pub async fn run(receiver: Receiver<()>, env: Arc<RwLock<Environment<'_>>>, setting: Setting) {
+pub async fn run(receiver: Receiver<()>, env: Arc<RwLock<Environment<'_>>>) {
     while let Ok(()) = receiver.recv_async().await {
         if let Environment::Loaded { runtime, .. } = &*env.read().await {
             let mut payloads = vec![Payload::default(); runtime.num_batch()];
             'run: loop {
-                if let Err(err) = runtime.process(&mut payloads, &setting).await {
+                if let Err(err) = runtime.process(&mut payloads).await {
                     log::error!("{}", err);
                     break 'run;
                 }