We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 7637cb9 commit 9ec05c9Copy full SHA for 9ec05c9
bindings/python/py_src/tokenizers/processors/__init__.pyi
@@ -98,8 +98,11 @@ class ByteLevel(PostProcessor):
98
Args:
99
trim_offsets (:obj:`bool`):
100
Whether to trim the whitespaces from the produced offsets.
101
+ add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
102
+ Whether to add a space to the first word if there isn't already one. This
103
+ lets us treat `hello` exactly like `say hello`.
104
"""
- def __init__(self, trim_offsets=True):
105
+ def __init__(self, trim_offsets=True, add_prefix_space=True):
106
pass
107
108
def num_special_tokens_to_add(self, is_pair):
0 commit comments