Switch to Nox, add build_dist script

Author: sethmlarson

.github/workflows/ci.yml

@@ -35,9 +35,9 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          python3.7 -m pip install tox
+          python3.7 -m pip install nox
       - name: Lint the code
-        run: tox -e lint
+        run: nox -s lint
 
   docs:
     runs-on: ubuntu-latest
@@ -50,9 +50,9 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          python3.7 -m pip install tox
+          python3.7 -m pip install nox
       - name: Build the docs
-        run: tox -e docs
+        run: nox -s docs
 
   test-linux:
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -30,7 +30,7 @@ To run the code generation make sure you have pre-requisites installed:
 Then you should be able to run the code generation by invoking:
 
 ```
-python utils/generate_api.py
+$ python utils/generate_api.py
 ```
 
 
@@ -45,10 +45,17 @@ The process for contributing to any of the Elasticsearch repositories is similar
    assure our users of the origin and continuing existence of the code. You only
    need to sign the CLA once.
 
-2. Run the test suite to ensure your changes do not break existing code:
+2. Run the linter and test suite to ensure your changes do not break existing code:
 
     ````
-    python setup.py test
+    # Install Nox for task management
+    $ python -m pip install nox
+    
+    # Auto-format and lint your changes
+    $ nox -s blacken
+   
+    # Run the test suite
+    $ python setup.py test
     ````
 
    See the README file in `test_elasticsearch` directory for more information on
@@ -68,4 +75,3 @@ The process for contributing to any of the Elasticsearch repositories is similar
 Then sit back and wait. There will probably be a discussion about the pull
 request and, if any changes are needed, we would love to work with you to get
 your pull request merged into elasticsearch-py.
-

noxfile.py

@@ -0,0 +1,49 @@
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+import nox
+
+
+SOURCE_FILES = (
+    "setup.py",
+    "noxfile.py",
+    "elasticsearch/",
+    "test_elasticsearch/",
+    "utils/",
+)
+
+
+@nox.session(python=["2.7", "3.4", "3.5", "3.6", "3.7", "3.8"])
+def test(session):
+    session.install(".")
+    session.install("-r", "dev-requirements.txt")
+
+    session.run("python", "setup.py", "test")
+
+
+@nox.session()
+def blacken(session):
+    session.install("black")
+
+    session.run("black", "--target-version=py27", *SOURCE_FILES)
+    session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
+
+    lint(session)
+
+
+@nox.session()
+def lint(session):
+    session.install("flake8", "black")
+
+    session.run("black", "--target-version=py27", "--check", *SOURCE_FILES)
+    session.run("flake8", *SOURCE_FILES)
+    session.run("python", "utils/license_headers.py", "check", *SOURCE_FILES)
+
+
+@nox.session()
+def docs(session):
+    session.install(".")
+    session.install("-rdev-requirements.txt", "sphinx-rtd-theme")
+
+    session.run("sphinx-build", "docs/", "docs/_build", "-b", "html")

tox.ini

@@ -0,0 +1,129 @@
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+"""A command line tool for building and verifying releases
+Can be used for building both 'elasticsearch' and 'elasticsearchX' dists.
+Only requires 'name' in 'setup.py' and the directory to be changed.
+"""
+
+import tempfile
+import os
+import shlex
+import re
+import contextlib
+import shutil
+
+
+base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+tmp_dir = None
+
+
+@contextlib.contextmanager
+def set_tmp_dir():
+    global tmp_dir
+    tmp_dir = tempfile.mkdtemp()
+    yield tmp_dir
+    shutil.rmtree(tmp_dir)
+    tmp_dir = None
+
+
+def run(*argv, expect_exit_code=0):
+    global tmp_dir
+    if tmp_dir is None:
+        os.chdir(base_dir)
+    else:
+        os.chdir(tmp_dir)
+
+    cmd = " ".join(shlex.quote(x) for x in argv)
+    print("$ " + cmd)
+    exit_code = os.system(cmd)
+    if exit_code != expect_exit_code:
+        print(
+            "Command exited incorrectly: should have been %d was %d"
+            % (expect_exit_code, exit_code)
+        )
+        exit(exit_code or 1)
+
+
+def test_dist(dist):
+    with set_tmp_dir() as tmp_dir:
+        dist_name = re.match(r"^(elasticsearch\d*)-", os.path.basename(dist)).group(1)
+
+        # Build the venv and install the dist
+        run("python", "-m", "venv", os.path.join(tmp_dir, "venv"))
+        venv_python = os.path.join(tmp_dir, "venv/bin/python")
+        run(venv_python, "-m", "pip", "install", "-U", "pip")
+        run(venv_python, "-m", "pip", "install", dist)
+
+        # Test the sync namespaces
+        run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
+        )
+        run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
+        )
+
+        # Ensure that async is not available yet
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name} import AsyncElasticsearch",
+            expect_exit_code=256,
+        )
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import async_scan, async_bulk, async_streaming_bulk, async_reindex",
+            expect_exit_code=256,
+        )
+
+        # Install aiohttp and see that async is now available
+        run(venv_python, "-m", "pip", "install", "aiohttp")
+        run(venv_python, "-c", f"from {dist_name} import AsyncElasticsearch")
+
+        # Ensure that the namespaces are correct for the dist
+        for suffix in ("", "1", "2", "5", "6", "7", "8", "9", "10"):
+            distx_name = f"elasticsearch{suffix}"
+            run(
+                venv_python,
+                "-c",
+                f"import {distx_name}",
+                expect_exit_code=256 if distx_name != dist_name else 0,
+            )
+
+        # Uninstall the dist, see that we can't import things anymore
+        run(venv_python, "-m", "pip", "uninstall", "--yes", dist_name)
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name} import Elasticsearch",
+            expect_exit_code=256,
+        )
+
+
+def main():
+    run("rm", "-rf", "build/", "dist/", "*.egg-info", ".eggs")
+    run("python", "setup.py", "sdist", "bdist_wheel")
+
+    for dist in os.listdir(os.path.join(base_dir, "dist")):
+        test_dist(os.path.join(base_dir, "dist", dist))
+
+    # After this run 'python -m twine upload dist/*'
+    print(
+        "\n\n"
+        "===============================\n\n"
+        "    * Releases are ready! *\n\n"
+        "$ python -m twine upload dist/*\n\n"
+        "==============================="
+    )
+
+
+if __name__ == "__main__":
+    main()

utils/Dockerfile

@@ -97,9 +97,7 @@ def parse_orig(self):
                             break
                 self.header = "\n".join(header_lines)
                 self.orders = re.findall(
-                    r'\n    (?:async )?def ([a-z_]+)\(',
-                    content,
-                    re.MULTILINE
+                    r"\n    (?:async )?def ([a-z_]+)\(", content, re.MULTILINE
                 )
 
     def _position(self, api):
@@ -154,7 +152,11 @@ def __init__(self, namespace, name, definition):
 
             # Try setting doc refs like 'current' and 'master' to our branches ref.
             if BRANCH_NAME is not None:
-                revised_url = re.sub("/elasticsearch/reference/[^/]+/", f"/elasticsearch/reference/{BRANCH_NAME}/", self.doc_url)
+                revised_url = re.sub(
+                    "/elasticsearch/reference/[^/]+/",
+                    f"/elasticsearch/reference/{BRANCH_NAME}/",
+                    self.doc_url,
+                )
                 if is_valid_url(revised_url):
                     self.doc_url = revised_url
                 else:
@@ -193,9 +195,11 @@ def params(self):
         return chain(
             ((p, parts[p]) for p in parts if parts[p]["required"]),
             (("body", self.body),) if self.body else (),
-            ((p, parts[p]) for p in parts
-             if not parts[p]["required"]
-             and p not in params),
+            (
+                (p, parts[p])
+                for p in parts
+                if not parts[p]["required"] and p not in params
+            ),
             sorted(params.items(), key=lambda x: (x[0] not in parts, x[0])),
         )
 
@@ -314,7 +318,7 @@ def dump_modules(modules):
         unasync.Rule(
             fromdir="/elasticsearch/_async/client/",
             todir="/elasticsearch/client/",
-            additional_replacements=additional_replacements
+            additional_replacements=additional_replacements,
         ),
     ]