Skip to content

Commit cc01186

Browse files
authored
Fix type notation of merges in BPE Python binding (#1766)
1 parent f1faec1 commit cc01186

File tree

3 files changed

+3
-3
lines changed

3 files changed

+3
-3
lines changed

bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class ByteLevelBPETokenizer(BaseTokenizer):
1616
def __init__(
1717
self,
1818
vocab: Optional[Union[str, Dict[str, int]]] = None,
19-
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
19+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
2020
add_prefix_space: bool = False,
2121
lowercase: bool = False,
2222
dropout: Optional[float] = None,

bindings/python/py_src/tokenizers/implementations/char_level_bpe.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class CharBPETokenizer(BaseTokenizer):
2525
def __init__(
2626
self,
2727
vocab: Optional[Union[str, Dict[str, int]]] = None,
28-
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
28+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
2929
unk_token: Union[str, AddedToken] = "<unk>",
3030
suffix: str = "</w>",
3131
dropout: Optional[float] = None,

bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class SentencePieceBPETokenizer(BaseTokenizer):
1616
def __init__(
1717
self,
1818
vocab: Optional[Union[str, Dict[str, int]]] = None,
19-
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
19+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
2020
unk_token: Union[str, AddedToken] = "<unk>",
2121
replacement: str = "▁",
2222
add_prefix_space: bool = True,

0 commit comments

Comments
 (0)