Minor bug fixes + add test cases + update readme

README-pypi.md

@@ -20,7 +20,7 @@ PyThaiNLP features include Thai word and subword segmentations, soundex, romaniz
 - thai2vec v0.2 - larger vocab, benchmarking results on Wongnai dataset
 - Sentiment classifier based on ULMFit and various product review datasets
 - Add ULMFit utility to PyThaiNLP
-- Add Thai romanization model thai2rom
+- Add Thai romanization model ThaiTransliterator
 - Retrain POS-tagging model
 - Improved word_tokenize (newmm, mm) and dict_word_tokenize
 - Documentation added

README.md

@@ -10,7 +10,7 @@
 
 Thai Natural Language Processing in Python.
 
-PyThaiNLP is a Python package for text processing and linguistic analysis, similar to `nltk`, but with focus on Thai language.
+PyThaiNLP is a Python package for text processing and linguistic analysis, similar to `nltk` but with focus on Thai language.
 
 PyThaiNLP supports Python 3.4+. Since version 1.7, PyThaiNLP deprecates its support for Python 2. Python 2 users can still use PyThaiNLP 1.6.
 
@@ -44,7 +44,7 @@ Development release
 $ pip install https://github.com/PyThaiNLP/pythainlp/archive/dev.zip
 ```
 
-Note: PyTorch is required for ulmfit sentiment analyser. ```pip install torch``` is needed for the feature.
+Note: PyTorch is required for ulmfit sentiment analyser. ```pip install torch``` is needed for the feature. gensim and keras packages may also needed for other modules that rely on these machine learning libraries.
 
 ## Documentation
 
@@ -103,7 +103,7 @@ $ pip install pythainlp
 $ pip install https://github.com/PyThaiNLP/pythainlp/archive/dev.zip
 ```
 
-หมายเหตุ: เนื่องจาก ulmfit sentiment analyser ต้องใช้ PyTorch จึงต้อง ```pip install torch``` เพื่อติดตั้ง PyTorhc ก่อน
+หมายเหตุ: เนื่องจาก ulmfit sentiment analyser ต้องใช้ PyTorch จึงต้อง ```pip install torch``` เพื่อติดตั้ง PyTorhc ก่อน มอดูลที่อาศัยการเรียนรู้ของเครื่องอื่นๆ อาจจำเป็นต้องติดตั้ง gensim และ keras ก่อนเช่นกัน
 
 ## เอกสารการใช้งาน
 

pythainlp/corpus/ttc.py

@@ -20,7 +20,7 @@ def word_freqs():
     ดึงข้อมูลความถี่คำของ Thai Textbook Corpus (TTC) มาใช้งาน
     โดยมีรูปแบบข้อมูลเป็น List[Tuple] [(word, frequency), ...]
     """
-    path = get_full_data_path("tnc_freq.txt")  # try local copy first
+    path = get_full_data_path("ttc_freq.txt")  # try local copy first
     if not os.path.exists(path):  # if fail, download from internet
         response = requests.get(_TCC_FREQ_URL)
         with open(path, "wb") as f:

pythainlp/sentiment/__init__.py

@@ -43,10 +43,12 @@ def sentiment(text, engine="old"):
             os.path.join(_SENTIMENT_PATH, "vocabulary.data"), "rb"
         ) as in_strm:
             vocabulary = dill.load(in_strm)
+
         with open(
             os.path.join(_SENTIMENT_PATH, "sentiment.data"), "rb"
         ) as in_strm:
             classifier = dill.load(in_strm)
+
         text = set(word_tokenize(text)) - _STOPWORDS
         featurized_test_sentence = {i: (i in text) for i in vocabulary}
 

pythainlp/ulmfit/utils.py

@@ -8,7 +8,6 @@
 from pythainlp.corpus import download, get_file
 from pythainlp.tokenize import word_tokenize
 
-
 try:
     import numpy as np
     from fastai.text import *
@@ -43,7 +42,6 @@ def __init__(self, engine="newmm"):
             * newmm - dictionary-based, Maximum Matching algorithm + TCC
             * longest - dictionary-based, Longest Matching
             * icu - use ICU, dictionary-based
-            * pylexto - use LexTo, dictionary-based
             * deepcut - use deepcut, language model-based
         """
         self.engine = engine

tests/__init__.py

@@ -7,6 +7,7 @@
     conceptnet,
     countries,
     provinces,
+    remove,
     thai_negations,
     thai_stopwords,
     thai_syllables,
@@ -66,6 +67,7 @@ def test_corpus(self):
         self.assertIsNotNone(thai_stopwords())
         self.assertIsNotNone(thai_syllables())
         self.assertIsNotNone(thai_words())
+        self.assertIsNotNone(remove("tnc_freq"))
 
     def test_tnc(self):
         self.assertIsNotNone(tnc.word_freqs())
@@ -150,6 +152,10 @@ def test_number(self):
             bahttext(5611116.50),
             "ห้าล้านหกแสนหนึ่งหมื่นหนึ่งพันหนึ่งร้อยสิบหกบาทห้าสิบสตางค์",
         )
+        self.assertEqual(
+            bahttext(116),
+            "หนึ่งร้อยสิบหกบาทถ้วน",
+        )
         self.assertEqual(num_to_thaiword(112), "หนึ่งร้อยสิบสอง")
         self.assertEqual(
             thaiword_to_num(
@@ -181,7 +187,7 @@ def test_romanization_royin(self):
 
     def test_sentiment(self):
         text = "เสียใจมาก"
-        # self.assertEqual(sentiment(text, engine="old"), "neg")
+        self.assertEqual(sentiment(text, engine="old"), "neg")
         # self.assertEqual(sentiment(text, engine="ulmfit"), "neg")
 
     # ### pythainlp.soundex