Merge pull request #667 from PyThaiNLP/Fix-666

Fixed #666

Author: wannaphong

pythainlp/tokenize/nercut.py

@@ -25,6 +25,7 @@ def segment(
         "DATE",
         "TIME",
     ],
+    tagger=_thainer
 ) -> List[str]:
     """
     Dictionary-based maximal matching word segmentation, constrained with
@@ -33,18 +34,17 @@ def segment(
 
     :param str text: text to be tokenized to words
     :parm list taglist: a list of named-entity tags to be used
+    :parm class tagger: ner tagger engine
     :return: list of words, tokenized from the text
     """
-    if not text or not isinstance(text, str):
+    if not isinstance(text, str):
         return []
 
-    global _thainer
-    tagged_words = _thainer.tag(text, pos=False)
+    tagged_words = tagger.tag(text, pos=False)
 
     words = []
     combining_word = ""
-    combining_word = ""
-    for curr_word, curr_tag in tagged_words:
+    for idx, (curr_word, curr_tag) in enumerate(tagged_words):
         if curr_tag != "O":
             tag = curr_tag[2:]
         else:
@@ -68,5 +68,15 @@ def segment(
         else:
             combining_word = ""
             words.append(curr_word)
+        if idx + 1 == len(tagged_words):
+            if (
+                curr_tag.startswith("B-") or curr_tag == "O"
+            ) and combining_word != "":
+                words.append(combining_word)
+                combining_word = ""
+                words.append(curr_word)
+            else:  # if tag is O
+                combining_word += curr_word
+                words.append(combining_word)
 
     return words

tests/test_tokenize.py

@@ -610,7 +610,9 @@ def test_nercut(self):
         self.assertEqual(nercut.segment(None), [])
         self.assertEqual(nercut.segment(""), [])
         self.assertIsNotNone(nercut.segment("ทดสอบ"))
-        self.assertIsNotNone(nercut.segment("ทดสอบ"))
+        self.assertIsNotNone(nercut.segment("ทุ๊กกโคนน"))
+        self.assertIsNotNone(nercut.segment("อือหือ"))
+        self.assertIsNotNone(nercut.segment("อย่าลืมอัพการ์ดนะจ๊ะ"))
         self.assertIsNotNone(word_tokenize("ทดสอบ", engine="nercut"))
 
     def test_ssg(self):