Merge pull request #958 from bact/mv-sent-tokenize-test

Move sent_tokenize with default crfcut to testx

Author: bact

tests/test_tokenize.py

@@ -217,8 +217,6 @@ def test_Tokenizer(self):
             Tokenizer(engine="catcut888")
 
     def test_sent_tokenize(self):
-        self.assertEqual(sent_tokenize(None), [])
-        self.assertEqual(sent_tokenize(""), [])
         self.assertEqual(
             sent_tokenize("รักน้ำ  รักปลา  ", engine="whitespace"),
             ["รักน้ำ", "รักปลา", ""],

tests/testx_tokenize.py

@@ -89,6 +89,10 @@ def testx_subword_tokenize(self):
         )
 
     def testx_sent_tokenize(self):
+        # Use default engine (crfcut)
+        self.assertEqual(sent_tokenize(None), [])
+        self.assertEqual(sent_tokenize(""), [])
+
         self.assertEqual(
             sent_tokenize(SENT_1, engine="crfcut"),
             SENT_1_TOKS,