Dry sample

docs/en/sampling_params.md

@@ -50,6 +50,12 @@ top_p: float = 1.0,
 top_k: int = -1,
 # Min-p sampling
 min_p: float = 0.0,
+# DRY sampling
+dry_multiplier: float = 0.0,
+dry_base: float = 0.0,
+dry_allowed_length: int = 2,
+dry_penalty_last_n: int = 0,
+dry_sequence_breakers: Optional[List[str]] = [],
 # Whether to ignore EOS token.
 ignore_eos: bool = False,
 # Whether to skip the special tokens during detokenization.

python/sglang/api.py

@@ -69,6 +69,12 @@ def gen(
     min_p: Optional[float] = None,
     frequency_penalty: Optional[float] = None,
     presence_penalty: Optional[float] = None,
+    # DRY sampling
+    dry_multiplier: float = 0.0,
+    dry_base: float = 0.0,
+    dry_allowed_length: int = 2,
+    dry_penalty_last_n: int = 0,
+    dry_sequence_breakers: Optional[List[str]] = [],
     ignore_eos: Optional[bool] = None,
     return_logprob: Optional[bool] = None,
     logprob_start_len: Optional[int] = None,
@@ -107,6 +113,11 @@ def gen(
         min_p,
         frequency_penalty,
         presence_penalty,
+        dry_multiplier,
+        dry_base,
+        dry_allowed_length,
+        dry_penalty_last_n,
+        dry_sequence_breakers,
         ignore_eos,
         return_logprob,
         logprob_start_len,
@@ -128,6 +139,11 @@ def gen_int(
     min_p: Optional[float] = None,
     frequency_penalty: Optional[float] = None,
     presence_penalty: Optional[float] = None,
+    dry_multiplier: float = 0.0,
+    dry_base: float = 0.0,
+    dry_allowed_length: int = 2,
+    dry_penalty_last_n: int = 0,
+    dry_sequence_breakers: Optional[List[str]] = [],
     ignore_eos: Optional[bool] = None,
     return_logprob: Optional[bool] = None,
     logprob_start_len: Optional[int] = None,
@@ -145,6 +161,11 @@ def gen_int(
         min_p,
         frequency_penalty,
         presence_penalty,
+        dry_multiplier,
+        dry_base,
+        dry_allowed_length,
+        dry_penalty_last_n,
+        dry_sequence_breakers,
         ignore_eos,
         return_logprob,
         logprob_start_len,
@@ -166,6 +187,11 @@ def gen_string(
     min_p: Optional[float] = None,
     frequency_penalty: Optional[float] = None,
     presence_penalty: Optional[float] = None,
+    dry_multiplier: float = 0.0,
+    dry_base: float = 0.0,
+    dry_allowed_length: int = 2,
+    dry_penalty_last_n: int = 0,
+    dry_sequence_breakers: Optional[List[str]] = [],
     ignore_eos: Optional[bool] = None,
     return_logprob: Optional[bool] = None,
     logprob_start_len: Optional[int] = None,
@@ -183,6 +209,11 @@ def gen_string(
         min_p,
         frequency_penalty,
         presence_penalty,
+        dry_multiplier,
+        dry_base,
+        dry_allowed_length,
+        dry_penalty_last_n,
+        dry_sequence_breakers,
         ignore_eos,
         return_logprob,
         logprob_start_len,

python/sglang/lang/compiler.py

@@ -1,7 +1,7 @@
 import multiprocessing
 from concurrent.futures import ThreadPoolExecutor
 from queue import Queue
-from typing import List, Union
+from typing import List, Union, Optional
 
 from sglang.global_config import global_config
 from sglang.lang.interpreter import ProgramState, StreamExecutor, cache_program
@@ -133,6 +133,11 @@ def run(
         min_p: float = 0.0,
         frequency_penalty: float = 0.0,
         presence_penalty: float = 0.0,
+        dry_multiplier: float = 0.0,
+        dry_base: float = 0.0,
+        dry_allowed_length: int = 2,
+        dry_penalty_last_n: int = 0,
+        dry_sequence_breakers: Optional[List[str]] = [],
         backend=None,
         **kwargs,
     ):
@@ -149,6 +154,11 @@ def run(
             min_p=min_p,
             frequency_penalty=frequency_penalty,
             presence_penalty=presence_penalty,
+            dry_multiplier=dry_multiplier,
+            dry_base=dry_base,
+            dry_allowed_length=dry_allowed_length,
+            dry_penalty_last_n=dry_penalty_last_n,
+            dry_sequence_breakers=dry_sequence_breakers,
         )
 
         return self.run_internal(backend, kwargs, default_sampling_para)
@@ -165,6 +175,11 @@ def run_batch(
         min_p: float = 0.0,
         frequency_penalty: float = 0.0,
         presence_penalty: float = 0.0,
+        dry_multiplier: float = 0.0,
+        dry_base: float = 0.0,
+        dry_allowed_length: int = 2,
+        dry_penalty_last_n: int = 0,
+        dry_sequence_breakers: Optional[List[str]] = [],
         backend=None,
         num_threads: Union[str, int] = "auto",
     ):
@@ -184,6 +199,11 @@ def run_batch(
             min_p=min_p,
             frequency_penalty=frequency_penalty,
             presence_penalty=presence_penalty,
+            dry_multiplier=dry_multiplier,
+            dry_base=dry_base,
+            dry_allowed_length=dry_allowed_length,
+            dry_penalty_last_n=dry_penalty_last_n,
+            dry_sequence_breakers=dry_sequence_breakers,
         )
 
         # Extract prefix by tracing and cache it

python/sglang/lang/interpreter.py

@@ -666,6 +666,11 @@ def _resolve_sampling_params(self, sampling_params):
             "min_p",
             "frequency_penalty",
             "presence_penalty",
+            "dry_multiplier",
+            "dry_base",
+            "dry_allowed_length",
+            "dry_penalty_last_n",
+            "dry_sequence_breakers",
             "ignore_eos",
             "return_logprob",
             "logprob_start_len",

python/sglang/lang/ir.py

@@ -25,6 +25,11 @@ class SglSamplingParams:
     min_p: float = 0.0
     frequency_penalty: float = 0.0
     presence_penalty: float = 0.0
+    dry_multiplier: float = 0.0
+    dry_base: float = 0.0
+    dry_allowed_length: int = 2
+    dry_penalty_last_n: int = 0
+    dry_sequence_breakers: Optional[List[str]] = ()
     ignore_eos: bool = False
     return_logprob: Optional[bool] = None
     logprob_start_len: Optional[int] = (None,)
@@ -46,6 +51,11 @@ def clone(self):
             self.min_p,
             self.frequency_penalty,
             self.presence_penalty,
+            self.dry_multiplier,
+            self.dry_base,
+            self.dry_allowed_length,
+            self.dry_penalty_last_n,
+            self.dry_sequence_breakers,
             self.ignore_eos,
             self.return_logprob,
             self.logprob_start_len,
@@ -62,8 +72,15 @@ def to_openai_kwargs(self):
             "stop": self.stop or None,
             "temperature": self.temperature,
             "top_p": self.top_p,
+            "min_p": self.min_p,
+            "top_k": self.top_k,
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
+            "dry_multiplier": self.dry_multiplier,
+            "dry_base": self.dry_base,
+            "dry_allowed_length": self.dry_allowed_length,
+            "dry_penalty_last_n": self.dry_penalty_last_n,
+            "dry_sequence_breakers": self.dry_sequence_breakers,
         }
 
     def to_vertexai_kwargs(self):
@@ -78,6 +95,12 @@ def to_vertexai_kwargs(self):
             "temperature": self.temperature,
             "top_p": self.top_p,
             "top_k": self.top_k if self.top_k > 0 else None,
+            "min_p": self.min_p,
+            "dry_multiplier": self.dry_multiplier,
+            "dry_base": self.dry_base,
+            "dry_allowed_length": self.dry_allowed_length,
+            "dry_penalty_last_n": self.dry_penalty_last_n,
+            "dry_sequence_breakers": self.dry_sequence_breakers,
         }
 
     def to_anthropic_kwargs(self):
@@ -106,6 +129,11 @@ def to_litellm_kwargs(self):
             "top_p": self.top_p,
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
+            "dry_multiplier": self.dry_multiplier,
+            "dry_base": self.dry_base,
+            "dry_allowed_length": self.dry_allowed_length,
+            "dry_penalty_last_n": self.dry_penalty_last_n,
+            "dry_sequence_breakers": self.dry_sequence_breakers,
         }
 
     def to_srt_kwargs(self):
@@ -119,6 +147,11 @@ def to_srt_kwargs(self):
             "min_p": self.min_p,
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
+            "dry_multiplier": self.dry_multiplier,
+            "dry_base": self.dry_base,
+            "dry_allowed_length": self.dry_allowed_length,
+            "dry_penalty_last_n": self.dry_penalty_last_n,
+            "dry_sequence_breakers": self.dry_sequence_breakers,
             "ignore_eos": self.ignore_eos,
             "regex": self.regex,
         }
@@ -155,6 +188,11 @@ def run(
         min_p: float = 0.0,
         frequency_penalty: float = 0.0,
         presence_penalty: float = 0.0,
+        dry_multiplier: float = 0.0,
+        dry_base: float = 0.0,
+        dry_allowed_length: int = 2,
+        dry_penalty_last_n: int = 0,
+        dry_sequence_breakers: Optional[List[str]] = [],
         ignore_eos: bool = False,
         return_logprob: Optional[bool] = None,
         logprob_start_len: Optional[int] = None,
@@ -176,6 +214,11 @@ def run(
             min_p=min_p,
             frequency_penalty=frequency_penalty,
             presence_penalty=presence_penalty,
+            dry_multiplier=dry_multiplier,
+            dry_base=dry_base,
+            dry_allowed_length=dry_allowed_length,
+            dry_penalty_last_n=dry_penalty_last_n,
+            dry_sequence_breakers=dry_sequence_breakers,
             ignore_eos=ignore_eos,
             return_logprob=return_logprob,
             logprob_start_len=logprob_start_len,
@@ -198,6 +241,11 @@ def run_batch(
         min_p: float = 0.0,
         frequency_penalty: float = 0.0,
         presence_penalty: float = 0.0,
+        dry_multiplier: float = 0.0,
+        dry_base: float = 0.0,
+        dry_allowed_length: int = 2,
+        dry_penalty_last_n: int = 0,
+        dry_sequence_breakers: Optional[List[str]] = [],
         ignore_eos: bool = False,
         return_logprob: Optional[bool] = None,
         logprob_start_len: Optional[int] = None,
@@ -237,6 +285,11 @@ def run_batch(
             min_p=min_p,
             frequency_penalty=frequency_penalty,
             presence_penalty=presence_penalty,
+            dry_multiplier=dry_multiplier,
+            dry_base=dry_base,
+            dry_allowed_length=dry_allowed_length,
+            dry_penalty_last_n=dry_penalty_last_n,
+            dry_sequence_breakers=dry_sequence_breakers,
             ignore_eos=ignore_eos,
             return_logprob=return_logprob,
             logprob_start_len=logprob_start_len,
@@ -418,6 +471,11 @@ def __init__(
         min_p: Optional[float] = None,
         frequency_penalty: Optional[float] = None,
         presence_penalty: Optional[float] = None,
+        dry_multiplier: float = 0.0,
+        dry_base: float = 0.0,
+        dry_allowed_length: int = 2,
+        dry_penalty_last_n: int = 0,
+        dry_sequence_breakers: Optional[List[str]] = [],
         ignore_eos: Optional[bool] = None,
         return_logprob: Optional[bool] = None,
         logprob_start_len: Optional[int] = None,
@@ -439,6 +497,11 @@ def __init__(
             min_p=min_p,
             frequency_penalty=frequency_penalty,
             presence_penalty=presence_penalty,
+            dry_multiplier=dry_multiplier,
+            dry_base=dry_base,
+            dry_allowed_length=dry_allowed_length,
+            dry_penalty_last_n=dry_penalty_last_n,
+            dry_sequence_breakers=dry_sequence_breakers,
             ignore_eos=ignore_eos,
             return_logprob=return_logprob,
             logprob_start_len=logprob_start_len,

python/sglang/srt/layers/sampler.py

@@ -29,7 +29,6 @@ def forward_cuda(self, logits: torch.Tensor, sampling_info: SamplingBatchInfo):
 
         if sampling_info.vocab_mask is not None:
             logits = logits.masked_fill(~sampling_info.vocab_mask, float("-inf"))
-
         logits = sampling_info.penalizer_orchestrator.apply(logits)
 
         probs = torch.softmax(logits, dim=-1)

python/sglang/srt/openai_api/adapter.py

@@ -433,6 +433,12 @@ def v1_generate_request(all_requests: List[CompletionRequest]):
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
                 "repetition_penalty": request.repetition_penalty,
+                "min_p": request.min_p,
+                "dry_multiplier": request.dry_multiplier,
+                "dry_base": request.dry_base,
+                "dry_allowed_length": request.dry_allowed_length,
+                "dry_penalty_last_n": request.dry_penalty_last_n,
+                "dry_sequence_breakers": request.dry_sequence_breakers,
                 "regex": request.regex,
                 "n": request.n,
                 "ignore_eos": request.ignore_eos,
@@ -798,9 +804,15 @@ def v1_chat_generate_request(
                 "stop": stop,
                 "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
+                "min_p": request.min_p,
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
                 "repetition_penalty": request.repetition_penalty,
+                "dry_multiplier": request.dry_multiplier,
+                "dry_base": request.dry_base,
+                "dry_allowed_length": request.dry_allowed_length,
+                "dry_penalty_last_n": request.dry_penalty_last_n,
+                "dry_sequence_breakers": request.dry_sequence_breakers,
                 "regex": request.regex,
                 "n": request.n,
             }

python/sglang/srt/openai_api/protocol.py

@@ -165,7 +165,12 @@ class CompletionRequest(BaseModel):
     min_tokens: Optional[int] = 0
     repetition_penalty: Optional[float] = 1.0
     stop_token_ids: Optional[List[int]] = Field(default_factory=list)
-
+    min_p: float = 0.0
+    dry_multiplier: float = 0.0
+    dry_base: float = 0.0
+    dry_allowed_length: int = 2
+    dry_penalty_last_n: int = 0
+    dry_sequence_breakers: Optional[List[str]] = []
 
 class CompletionResponseChoice(BaseModel):
     index: int
@@ -265,7 +270,12 @@ class ChatCompletionRequest(BaseModel):
     min_tokens: Optional[int] = 0
     repetition_penalty: Optional[float] = 1.0
     stop_token_ids: Optional[List[int]] = Field(default_factory=list)
-
+    min_p: float = 0.0
+    dry_multiplier: float = 0.0
+    dry_base: float = 0.0
+    dry_allowed_length: int = 2
+    dry_penalty_last_n: int = 0
+    dry_sequence_breakers: Optional[List[str]] = []
 
 class ChatMessage(BaseModel):
     role: Optional[str] = None

python/sglang/srt/sampling/penaltylib/__init__.py

@@ -3,11 +3,13 @@
 from .penalizers.min_new_tokens import BatchedMinNewTokensPenalizer
 from .penalizers.presence_penalty import BatchedPresencePenalizer
 from .penalizers.repetition_penalty import BatchedRepetitionPenalizer
+from .penalizers.dry_penalty import BatchedDryPenalizer
 
 __all__ = [
     "BatchedFrequencyPenalizer",
     "BatchedMinNewTokensPenalizer",
     "BatchedPresencePenalizer",
     "BatchedRepetitionPenalizer",
     "BatchedPenalizerOrchestrator",
+    "BatchedDryPenalizer",
 ]