Skip to content

Clean code (/corpus, /tools, /ulmfit) #132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 20, 2018
Merged

Clean code (/corpus, /tools, /ulmfit) #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7a1b935
Clean code
bact Oct 20, 2018
a7363d3
put corpus database url into CORPUS_DB_URL constant
bact Oct 20, 2018
7d71561
use "not x" instead of "len(x) == 0"
bact Oct 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 93 additions & 47 deletions pythainlp/corpus/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,40 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import,unicode_literals
from pythainlp.tools import get_path_db,get_path_data
from tinydb import TinyDB,Query
from future.moves.urllib.request import urlopen
from tqdm import tqdm
import requests

from __future__ import absolute_import, unicode_literals

import os

import requests
#__all__ = ["thaipos", "thaiword","alphabet","tone","country","wordnet"]
path_db_=get_path_db()
from future.moves.urllib.request import urlopen
from pythainlp.tools import get_path_data, get_path_db
from tinydb import Query, TinyDB
from tqdm import tqdm

CORPUS_DB_URL = (
"https://raw.githubusercontent.com/PyThaiNLP/pythainlp-corpus/master/db.json"
)

# __all__ = ["thaipos", "thaiword","alphabet","tone","country","wordnet"]
path_db_ = get_path_db()


def get_file(name):
db=TinyDB(path_db_)
db = TinyDB(path_db_)
temp = Query()
if len(db.search(temp.name==name))>0:
path= get_path_data(db.search(temp.name==name)[0]['file'])
if len(db.search(temp.name == name)) > 0:
path = get_path_data(db.search(temp.name == name)[0]["file"])
db.close()
if not os.path.exists(path):
download(name)
return path


def download_(url, dst):
"""
@param: url to download file
@param: dst place to put the file
"""
file_size = int(urlopen(url).info().get('Content-Length', -1))
file_size = int(urlopen(url).info().get("Content-Length", -1))
if os.path.exists(dst):
first_byte = os.path.getsize(dst)
else:
Expand All @@ -32,55 +43,90 @@ def download_(url, dst):
return file_size
header = {"Range": "bytes=%s-%s" % (first_byte, file_size)}
pbar = tqdm(
total=file_size, initial=first_byte,
unit='B', unit_scale=True, desc=url.split('/')[-1])
total=file_size,
initial=first_byte,
unit="B",
unit_scale=True,
desc=url.split("/")[-1],
)
req = requests.get(url, headers=header, stream=True)
with(open(get_path_data(dst), 'wb')) as f:
with (open(get_path_data(dst), "wb")) as f:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
pbar.update(1024)
pbar.close()
#return file_size
def download(name,force=False):
db=TinyDB(path_db_)
# return file_size


def download(name, force=False):
db = TinyDB(path_db_)
temp = Query()
data=requests.get("https://raw.githubusercontent.com/PyThaiNLP/pythainlp-corpus/master/db.json")
data_json=data.json()
data = requests.get(CORPUS_DB_URL)
data_json = data.json()
if name in list(data_json.keys()):
temp_name=data_json[name]
print("Download : "+name)
if len(db.search(temp.name==name))==0:
print(name+" "+temp_name['version'])
download_(temp_name['download'],temp_name['file_name'])
db.insert({'name': name, 'version': temp_name['version'],'file':temp_name['file_name']})
temp_name = data_json[name]
print("Download : " + name)

if not db.search(temp.name == name):
print(name + " " + temp_name["version"])
download_(temp_name["download"], temp_name["file_name"])
db.insert(
{
"name": name,
"version": temp_name["version"],
"file": temp_name["file_name"],
}
)
else:
if len(db.search(temp.name==name and temp.version==temp_name['version']))==0:
if not db.search(
temp.name == name and temp.version == temp_name["version"]
):
print("have update")
print("from "+name+" "+db.search(temp.name==name)[0]['version']+" update to "+name+" "+temp_name['version'])
yes_no="y"
if force==False:
yes_no=str(input("y or n : ")).lower()
if "y"==yes_no:
download_(temp_name['download'],temp_name['file_name'])
db.update({'version':temp_name['version']},temp.name==name)
print(
"from "
+ name
+ " "
+ db.search(temp.name == name)[0]["version"]
+ " update to "
+ name
+ " "
+ temp_name["version"]
)
yes_no = "y"
if not force:
yes_no = str(input("y or n : ")).lower()
if "y" == yes_no:
download_(temp_name["download"], temp_name["file_name"])
db.update({"version": temp_name["version"]}, temp.name == name)
else:
print("re-download")
print("from "+name+" "+db.search(temp.name==name)[0]['version']+" update to "+name+" "+temp_name['version'])
yes_no="y"
if force==False:
yes_no=str(input("y or n : ")).lower()
if "y"==yes_no:
download_(temp_name['download'],temp_name['file_name'])
db.update({'version':temp_name['version']},temp.name==name)
print(
"from "
+ name
+ " "
+ db.search(temp.name == name)[0]["version"]
+ " update to "
+ name
+ " "
+ temp_name["version"]
)
yes_no = "y"
if not force:
yes_no = str(input("y or n : ")).lower()
if "y" == yes_no:
download_(temp_name["download"], temp_name["file_name"])
db.update({"version": temp_name["version"]}, temp.name == name)
db.close()


def remove(name):
db=TinyDB(path_db_)
db = TinyDB(path_db_)
temp = Query()
data=db.search(temp.name==name)
if len(data)>0:
path=get_file(name)
data = db.search(temp.name == name)
if len(data) > 0:
path = get_file(name)
os.remove(path)
db.remove(temp.name==name)
db.remove(temp.name == name)
return True
return False
return False
37 changes: 22 additions & 15 deletions pythainlp/tools/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,32 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import,unicode_literals

from __future__ import absolute_import, unicode_literals

import os
import dill
from pythainlp.tokenize import tcc
import marisa_trie
import subprocess
import sys


def install_package(package):
subprocess.call([sys.executable, "-m", "pip", "install", package])


def get_path_db():
path = os.path.join(get_path_pythainlp_data(), "db.json")
if not os.path.exists(path):
from tinydb import TinyDB
db=TinyDB(path)
#db.insert({'name': 'hi', 'version': '0.1','file':''})
return path
path = os.path.join(get_path_pythainlp_data(), "db.json")
if not os.path.exists(path):
from tinydb import TinyDB

db = TinyDB(path)
# db.insert({'name': 'hi', 'version': '0.1','file':''})
return path


def get_path_data(filename):
return os.path.join(get_path_pythainlp_data(), filename)
return os.path.join(get_path_pythainlp_data(), filename)


def get_path_pythainlp_data():
path= os.path.join(os.path.expanduser("~"), 'pythainlp-data')
if not os.path.exists(path):
os.makedirs(path)
return path
path = os.path.join(os.path.expanduser("~"), "pythainlp-data")
if not os.path.exists(path):
os.makedirs(path)
return path
3 changes: 2 additions & 1 deletion pythainlp/ulmfit/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import,unicode_literals

from __future__ import absolute_import, unicode_literals
Loading