download json schemas recursively

this should make us a bit more resilient against renaming/moving
of JSON schema files in apm-server

Author: beniwohli

Makefile

@@ -19,6 +19,6 @@ docs:
 	bash ./scripts/build_docs.sh apm-agent-python ./docs ${BUILD_DIR}
 
 update-json-schema:
-	bash ./tests/scripts/download_json_schema.sh
+	python3 ./tests/scripts/download_json_schema.py master tests/.schemacache
 
 .PHONY: isort flake8 test coverage docs update-json-schema

tests/scripts/download_json_schema.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+from urllib import request
+
+BASE_URL = "https://api.github.com/repos/elastic/apm-server/contents/{path}?ref={branch}"
+
+
+def download_folder(name, branch, base_folder):
+    response = request.urlopen(BASE_URL.format(path=name, branch=branch))
+    data = json.loads(response.read().decode("utf-8"))
+    for item in data:
+        if item["type"] == "dir":
+            download_folder(item["path"], branch, base_folder)
+        elif item["type"] == "file":
+            path = base_folder + item["path"][9:]
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            request.urlretrieve(item["download_url"], base_folder + item["path"][9:])
+            print("downloaded {} to {}".format(item["path"][9:], os.path.dirname(path)))
+
+
+if __name__ == "__main__":
+    path = os.path.abspath(sys.argv[2])
+    download_folder("docs/spec", sys.argv[1], path)