Skip to content

Commit 9ec05c9

Browse files
committed
add prefix space to pyi
1 parent 7637cb9 commit 9ec05c9

File tree

1 file changed

+4
-1
lines changed

1 file changed

+4
-1
lines changed

bindings/python/py_src/tokenizers/processors/__init__.pyi

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,11 @@ class ByteLevel(PostProcessor):
9898
Args:
9999
trim_offsets (:obj:`bool`):
100100
Whether to trim the whitespaces from the produced offsets.
101+
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
102+
Whether to add a space to the first word if there isn't already one. This
103+
lets us treat `hello` exactly like `say hello`.
101104
"""
102-
def __init__(self, trim_offsets=True):
105+
def __init__(self, trim_offsets=True, add_prefix_space=True):
103106
pass
104107

105108
def num_special_tokens_to_add(self, is_pair):

0 commit comments

Comments
 (0)