Update Peter Norvig's spell checker to suggest words based on probability

pythainlp/corpus/__init__.py

@@ -3,16 +3,16 @@
 import os
 
 import requests
-from future.moves.urllib.request import urlopen
 from pythainlp.tools import get_path_data, get_path_db
 from tinydb import Query, TinyDB
 from tqdm import tqdm
+from urllib.request import urlopen
 
 CORPUS_DB_URL = (
     "https://raw.githubusercontent.com/PyThaiNLP/pythainlp-corpus/master/db.json"
 )
 
-# __all__ = ["thaipos", "thaiword","alphabet","tone","country","wordnet"]
+# __all__ = ["thaipos", "thaiword", "alphabet", "tone", "country", "wordnet"]
 path_db_ = get_path_db()
 
 

pythainlp/corpus/tnc.py

@@ -57,6 +57,6 @@ def get_word_frequency_all():
     listword = []
     for line in lines:
         listindata = line.split("	")
-        listword.append((listindata[0], listindata[1]))
+        listword.append((listindata[0], int(listindata[1])))
 
     return listword

pythainlp/corpus/ttc.py

@@ -34,6 +34,6 @@ def get_word_frequency_all():
     listword = []
     for line in lines:
         listindata = line.split("	")
-        listword.append((listindata[0], listindata[1]))
+        listword.append((listindata[0], int(listindata[1])))
 
     return listword

pythainlp/spell/pn.py

@@ -1,30 +1,32 @@
 # -*- coding: utf-8 -*-
 """
-Spell checker
+Spell checker, using Peter Norvig algorithm + word frequency from Thai National Corpus
 
-Based on Peter Norvig's Python code at http://norvig.com/spell-correct.html
+Based on Peter Norvig's Python code from http://norvig.com/spell-correct.html
 """
 from collections import Counter
-from pythainlp.corpus.thaiword import get_data
 
-WORDS = Counter(get_data())
+from pythainlp.corpus import tnc
 
+WORDS = Counter(dict(tnc.get_word_frequency_all()))
+WORDS_TOTAL = sum(WORDS.values())
 
-def prob(word, n=sum(WORDS.values())):
+
+def _prob(word, n=WORDS_TOTAL):
     "Probability of `word`."
     return WORDS[word] / n
 
 
-def correction(word):
+def _correction(word):
     "แสดงคำที่เป็นไปได้มากที่สุด"
-    return max(spell(word), key=prob)
+    return max(spell(word), key=_prob)
 
 
-def known(words):
+def _known(words):
     return list(w for w in words if w in WORDS)
 
 
-def edits1(word):
+def _edits1(word):
     letters = [
         "ก",
         "ข",
@@ -111,12 +113,12 @@ def edits1(word):
     return set(deletes + transposes + replaces + inserts)
 
 
-def edits2(word):
-    return (e2 for e1 in edits1(word) for e2 in edits1(e1))
+def _edits2(word):
+    return (e2 for e1 in _edits1(word) for e2 in _edits1(e1))
 
 
 def spell(word):
     if not word:
         return ""
-    else:
-        return known([word]) or known(edits1(word)) or known(edits2(word)) or [word]
+
+    return _known([word]) or _known(_edits1(word)) or _known(_edits2(word)) or [word]