fix(WordTokenizer): adjusted token list assigned to token list returned by tokenize method

kaldan007 · kaldan007 · commit dc898a9786a7 · 2021-03-22T14:39:45.000+05:30
diff --git a/botok/tokenizers/wordtokenizer.py b/botok/tokenizers/wordtokenizer.py
@@ -88,7 +88,7 @@ def tokenize(self, string, split_affixes=True, spaces_as_punct=False, debug=Fals
         MergeDagdra().merge(tokens)
 
         # do adjustments
-        self.adj.adjust(tokens)
+        tokens = self.adj.adjust(tokens)
 
         return tokens
 
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/modifytokens/test_matchers.py b/tests/modifytokens/test_matchers.py
@@ -5,6 +5,7 @@
 
 from botok import *
 
+
 input_str = " མཐའི་རྒྱ་མཚོའི་གླིང་། ཤི་བཀྲ་ཤིས་  tr བདེ་་ལེ གས། བཀྲ་ཤིས་བདེ་ལེགས་ཀཀ"
 
 
diff --git a/tests/test_bugs.py b/tests/test_bugs.py
@@ -5,6 +5,7 @@
 
 from botok import TokChunks, Config, Trie, BoSyl, Tokenize, Chunks, ChunkFramework
 
+
 sys.path.append("../")
 
 
diff --git a/tests/tokenizers/test_bugs_missing_tokens.py b/tests/tokenizers/test_bugs_missing_tokens.py
@@ -1,6 +1,7 @@
 from botok import *
 
 
+
 def test_missing_token1(wt):
     input_str = "འཐུང་བུད་"
     tokens = wt.tokenize(input_str, split_affixes=False)
diff --git a/tests/tokenizers/test_sent_par_tokenizer.py b/tests/tokenizers/test_sent_par_tokenizer.py
@@ -1,6 +1,7 @@
 import pytest
 
-from botok import WordTokenizer, paragraph_tokenizer, sentence_tokenizer
+from botok import paragraph_tokenizer, sentence_tokenizer
+
 
 text = (
     "བཀུར་བར་མི་འགྱུར་ཞིང༌། །བརྙས་བཅོས་མི་སྙན་རྗོད་པར་བྱེད། །དབང་དང་འབྱོར་པ་ལྡན་པ་ཡི། །རྒྱལ་རིགས་ཕལ་ཆེར་བག་མེད་པས། །"

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`
`6`	`6`	`from botok import *`
`7`	`7`
	`8`	`+`
`8`	`9`	`input_str = " མཐའི་རྒྱ་མཚོའི་གླིང་། ཤི་བཀྲ་ཤིས་ tr བདེ་་ལེ གས། བཀྲ་ཤིས་བདེ་ལེགས་ཀཀ"`
`9`	`10`
`10`	`11`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`
`6`	`6`	`from botok import TokChunks, Config, Trie, BoSyl, Tokenize, Chunks, ChunkFramework`
`7`	`7`
	`8`	`+`
`8`	`9`	`sys.path.append("../")`
`9`	`10`
`10`	`11`