Skip to content

Commit 03bb3bf

Browse files
committed
Move sent_tokenize with default crfcut to testx
1 parent 2283d9d commit 03bb3bf

File tree

2 files changed

+4
-2
lines changed

2 files changed

+4
-2
lines changed

tests/test_tokenize.py

-2
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,6 @@ def test_Tokenizer(self):
217217
Tokenizer(engine="catcut888")
218218

219219
def test_sent_tokenize(self):
220-
self.assertEqual(sent_tokenize(None), [])
221-
self.assertEqual(sent_tokenize(""), [])
222220
self.assertEqual(
223221
sent_tokenize("รักน้ำ รักปลา ", engine="whitespace"),
224222
["รักน้ำ", "รักปลา", ""],

tests/testx_tokenize.py

+4
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ def testx_subword_tokenize(self):
8989
)
9090

9191
def testx_sent_tokenize(self):
92+
# Use default engine (crfcut)
93+
self.assertEqual(sent_tokenize(None), [])
94+
self.assertEqual(sent_tokenize(""), [])
95+
9296
self.assertEqual(
9397
sent_tokenize(SENT_1, engine="crfcut"),
9498
SENT_1_TOKS,

0 commit comments

Comments
 (0)