Update other non-Thai characters in newmm

Author: wannaphong

pythainlp/tokenize/newmm.py

@@ -44,7 +44,7 @@
 \d+([,\.]\d+)*|    # numbers
 [ \t]+|            # spaces
 \r?\n|             # newlines
-[^\u0E00-\u0E7F]+  # other non-Thai characters
+[^\u0E00-\u0E7F \t]+  # other non-Thai characters
 """
 )
 

tests/test_tokenize.py

@@ -653,6 +653,15 @@ def test_newmm(self):
                 keep_whitespace=False,
             )
         )
+        self.assertEqual(
+            word_tokenize("(คนไม่เอา)", engine="newmm"), ['(', 'คน', 'ไม่', 'เอา', ')']
+        )
+        self.assertEqual(
+            word_tokenize("กม/ชม", engine="newmm"), ['กม', '/', 'ชม']
+        )
+        self.assertEqual(
+            word_tokenize("สีหน้า(รถ)", engine="newmm"), ['สีหน้า', '(', 'รถ', ')']
+        )
 
     def test_newmm_longtext(self):
         self.assertIsInstance(