implemented a caching mechanism to speed up debugging.

Author: pdamoc

cache.py

@@ -0,0 +1,20 @@
+import json
+import requests  
+import os
+
+if os.path.exists("cache.json"):
+    with open("cache.json") as fo:
+        cache = json.loads(fo.read())
+else:
+    cache = {}
+
+def fetch(url, isJSON=True):
+    if not url in cache:
+        if isJSON:
+            cache[url] = requests.get(url).json()
+        else: 
+            cache[url] = requests.get(url).text
+            
+        with open("cache.json", "w") as fo:
+            fo.write(json.dumps(cache))
+    return cache[url]

debug.py

@@ -2,16 +2,17 @@
 from templates import indexTemplate, pkgTemplate, moduleTemplate, toHtml
 import requests 
 import re
+from cache import fetch
 
 from generate import Module
 
 def debug_module(pkg_name, module_name):
-    all_pkgs = requests.get("http://package.elm-lang.org/all-packages").json()
+    all_pkgs = fetch("http://package.elm-lang.org/all-packages")
     all_pkgs_dict = {p["name"]:p for p in all_pkgs}
     pkg_data = all_pkgs_dict[pkg_name]
 
     jsonURL = "/".join(["http://package.elm-lang.org/packages", pkg_name, pkg_data["versions"][0], "documentation.json"])
-    json_data = requests.get(jsonURL).json()
+    json_data = fetch(jsonURL)
     json_data_dict = {m["name"]:m for m in json_data}
 
 

generate.py

@@ -4,7 +4,7 @@
 import sqlite3
 
 import json
-import requests
+from cache import fetch
 
 from cgi import escape
 
@@ -243,10 +243,10 @@ def docname(pkg, module=None):
 def generate_all():
     global pkgs
     print("feching all packages list ..."),
-    all_pkgs = requests.get(pkgsURL+"all-packages").json()
+    all_pkgs = fetch(pkgsURL+"all-packages")
     print("DONE!")
     print("feching new packages list ..."),
-    new_pkgs = requests.get(pkgsURL+"new-packages").json()
+    new_pkgs = fetch(pkgsURL+"new-packages")
     print("DONE!")
 
     new_pkgs = list(set(new_pkgs))
@@ -270,7 +270,7 @@ def generate_all():
         print "Generating package: "+pkg_name+" [% 3d / %03d]..."%(idx, no_pkgs), 
 
         docURL = pkgsURL+"/".join(["packages", pkg_name, pkg_version, "documentation"])+".json"
-        json = requests.get(docURL).json()
+        json = fetch(docURL)
         # module = Module(json)
         links = []
         for module in json:
@@ -291,7 +291,7 @@ def generate_all():
         print "DONE!"
 
 DEBUG = False 
-DEBUG = True
+# DEBUG = True
 
 if __name__ == '__main__':
     print("starting ...")

templates.py

@@ -5,7 +5,7 @@
 import markdown
 import markdown.extensions.fenced_code
 
-import requests
+from cache import fetch
 
 gitHub = "https://github.com/"
 gitHubRaw = "https://raw.githubusercontent.com"
@@ -101,7 +101,7 @@ def moduleslist(this, options, items):
 
 
 def gitRM(this, name):
-  readme = requests.get("/".join([gitHub, name, "raw/master", "README.md"])).text
+  readme = fetch("/".join([gitHub, name, "raw/master", "README.md"]), False)
   result = toHtml(readme)
   return result