fix(trie): non inflectible word type error bug fix

Author: kaldan007

botok/tries/trie.py

@@ -120,8 +120,8 @@ def add_non_inflectible(self, word):
         if not syls:
             return None
 
-        infl = self.__join_syls(syls)
-        self.add(infl)
+        # infl = self.__join_syls(syls)
+        self.add(syls)
 
     def inflect_n_modify_trie(self, word, deactivate=False, skrt=False):
         """

tests/tokenizers/test_wordtokenizer.py

@@ -126,3 +126,8 @@ def test_spaces_as_punct(wt):
     assert tokens[1].text == " "
     assert tokens[2].text == "ཀྲ་"
     assert tokens[8].text == " \n"
+
+def test_particle_bug(wt):
+    input_str = "བོད་གིས"
+    tokens = wt.tokenize(input_str)
+    assert tokens[1].pos == "PART"