Merge pull request #48 from pypa/feature/build-meta-command

New command pep517.meta

Author: takluyver

dev-requirements.txt

@@ -4,3 +4,5 @@ mock
 testpath
 pytoml
 setuptools>=30
+importlib_metadata
+zipp

pep517/build.py

@@ -3,25 +3,56 @@
 import argparse
 import logging
 import os
-import contextlib
 import pytoml
 import shutil
-import errno
-import tempfile
 
 from .envbuild import BuildEnvironment
 from .wrappers import Pep517HookCaller
+from .dirtools import tempdir, mkdir_p
+from .compat import FileNotFoundError
 
 log = logging.getLogger(__name__)
 
 
-@contextlib.contextmanager
-def tempdir():
-    td = tempfile.mkdtemp()
+def validate_system(system):
+    """
+    Ensure build system has the requisite fields.
+    """
+    required = {'requires', 'build-backend'}
+    if not (required <= set(system)):
+        message = "Missing required fields: {missing}".format(
+            missing=required-set(system),
+        )
+        raise ValueError(message)
+
+
+def load_system(source_dir):
+    """
+    Load the build system from a source dir (pyproject.toml).
+    """
+    pyproject = os.path.join(source_dir, 'pyproject.toml')
+    with open(pyproject) as f:
+        pyproject_data = pytoml.load(f)
+    return pyproject_data['build-system']
+
+
+def compat_system(source_dir):
+    """
+    Given a source dir, attempt to get a build system backend
+    and requirements from pyproject.toml. Fallback to
+    setuptools but only if the file was not found or a build
+    system was not indicated.
+    """
     try:
-        yield td
-    finally:
-        shutil.rmtree(td)
+        system = load_system(source_dir)
+    except (FileNotFoundError, KeyError):
+        system = {}
+    system.setdefault(
+        'build-backend',
+        'setuptools.build_meta:__legacy__',
+    )
+    system.setdefault('requires', ['setuptools', 'wheel'])
+    return system
 
 
 def _do_build(hooks, env, dist, dest):
@@ -42,33 +73,16 @@ def _do_build(hooks, env, dist, dest):
         shutil.move(source, os.path.join(dest, os.path.basename(filename)))
 
 
-def mkdir_p(*args, **kwargs):
-    """Like `mkdir`, but does not raise an exception if the
-    directory already exists.
-    """
-    try:
-        return os.mkdir(*args, **kwargs)
-    except OSError as exc:
-        if exc.errno != errno.EEXIST:
-            raise
-
-
-def build(source_dir, dist, dest=None):
-    pyproject = os.path.join(source_dir, 'pyproject.toml')
+def build(source_dir, dist, dest=None, system=None):
+    system = system or load_system(source_dir)
     dest = os.path.join(source_dir, dest or 'dist')
     mkdir_p(dest)
 
-    with open(pyproject) as f:
-        pyproject_data = pytoml.load(f)
-    # Ensure the mandatory data can be loaded
-    buildsys = pyproject_data['build-system']
-    requires = buildsys['requires']
-    backend = buildsys['build-backend']
-
-    hooks = Pep517HookCaller(source_dir, backend)
+    validate_system(system)
+    hooks = Pep517HookCaller(source_dir, system['build-backend'])
 
     with BuildEnvironment() as env:
-        env.pip_install(requires)
+        env.pip_install(system['requires'])
         _do_build(hooks, env, dist, dest)
 
 

pep517/compat.py

@@ -1,7 +1,10 @@
-"""Handle reading and writing JSON in UTF-8, on Python 3 and 2."""
+"""Python 2/3 compatibility"""
 import json
 import sys
 
+
+# Handle reading and writing JSON in UTF-8, on Python 3 and 2.
+
 if sys.version_info[0] >= 3:
     # Python 3
     def write_json(obj, path, **kwargs):
@@ -21,3 +24,11 @@ def write_json(obj, path, **kwargs):
     def read_json(path):
         with open(path, 'rb') as f:
             return json.load(f)
+
+
+# FileNotFoundError
+
+try:
+    FileNotFoundError = FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError

pep517/dirtools.py

@@ -0,0 +1,44 @@
+import os
+import io
+import contextlib
+import tempfile
+import shutil
+import errno
+import zipfile
+
+
+@contextlib.contextmanager
+def tempdir():
+    """Create a temporary directory in a context manager."""
+    td = tempfile.mkdtemp()
+    try:
+        yield td
+    finally:
+        shutil.rmtree(td)
+
+
+def mkdir_p(*args, **kwargs):
+    """Like `mkdir`, but does not raise an exception if the
+    directory already exists.
+    """
+    try:
+        return os.mkdir(*args, **kwargs)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            raise
+
+
+def dir_to_zipfile(root):
+    """Construct an in-memory zip file for a directory."""
+    buffer = io.BytesIO()
+    zip_file = zipfile.ZipFile(buffer, 'w')
+    for root, dirs, files in os.walk(root):
+        for path in dirs:
+            fs_path = os.path.join(root, path)
+            rel_path = os.path.relpath(fs_path, root)
+            zip_file.writestr(rel_path + '/', '')
+        for path in files:
+            fs_path = os.path.join(root, path)
+            rel_path = os.path.relpath(fs_path, root)
+            zip_file.write(fs_path, rel_path)
+    return zip_file

pep517/envbuild.py

@@ -10,7 +10,7 @@
 from sysconfig import get_paths
 from tempfile import mkdtemp
 
-from .wrappers import Pep517HookCaller
+from .wrappers import Pep517HookCaller, LoggerWrapper
 
 log = logging.getLogger(__name__)
 
@@ -90,9 +90,14 @@ def pip_install(self, reqs):
         if not reqs:
             return
         log.info('Calling pip to install %s', reqs)
-        check_call([
+        cmd = [
             sys.executable, '-m', 'pip', 'install', '--ignore-installed',
-            '--prefix', self.path] + list(reqs))
+            '--prefix', self.path] + list(reqs)
+        check_call(
+            cmd,
+            stdout=LoggerWrapper(log, logging.INFO),
+            stderr=LoggerWrapper(log, logging.ERROR),
+        )
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         needs_cleanup = (

pep517/meta.py

@@ -0,0 +1,90 @@
+"""Build metadata for a project using PEP 517 hooks.
+"""
+import argparse
+import logging
+import os
+import shutil
+import functools
+
+try:
+    import importlib.metadata as imp_meta
+except ImportError:
+    import importlib_metadata as imp_meta
+
+try:
+    from zipfile import Path
+except ImportError:
+    from zipp import Path
+
+from .envbuild import BuildEnvironment
+from .wrappers import Pep517HookCaller, quiet_subprocess_runner
+from .dirtools import tempdir, mkdir_p, dir_to_zipfile
+from .build import validate_system, load_system, compat_system
+
+log = logging.getLogger(__name__)
+
+
+def _prep_meta(hooks, env, dest):
+    reqs = hooks.get_requires_for_build_wheel({})
+    log.info('Got build requires: %s', reqs)
+
+    env.pip_install(reqs)
+    log.info('Installed dynamic build dependencies')
+
+    with tempdir() as td:
+        log.info('Trying to build metadata in %s', td)
+        filename = hooks.prepare_metadata_for_build_wheel(td, {})
+        source = os.path.join(td, filename)
+        shutil.move(source, os.path.join(dest, os.path.basename(filename)))
+
+
+def build(source_dir='.', dest=None, system=None):
+    system = system or load_system(source_dir)
+    dest = os.path.join(source_dir, dest or 'dist')
+    mkdir_p(dest)
+    validate_system(system)
+    hooks = Pep517HookCaller(source_dir, system['build-backend'])
+
+    with hooks.subprocess_runner(quiet_subprocess_runner):
+        with BuildEnvironment() as env:
+            env.pip_install(system['requires'])
+            _prep_meta(hooks, env, dest)
+
+
+def build_as_zip(builder=build):
+    with tempdir() as out_dir:
+        builder(dest=out_dir)
+        return dir_to_zipfile(out_dir)
+
+
+def load(root):
+    """
+    Given a source directory (root) of a package,
+    return an importlib.metadata.Distribution object
+    with metadata build from that package.
+    """
+    root = os.path.expanduser(root)
+    system = compat_system(root)
+    builder = functools.partial(build, source_dir=root, system=system)
+    path = Path(build_as_zip(builder))
+    return imp_meta.PathDistribution(path)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    'source_dir',
+    help="A directory containing pyproject.toml",
+)
+parser.add_argument(
+    '--out-dir', '-o',
+    help="Destination in which to save the builds relative to source dir",
+)
+
+
+def main():
+    args = parser.parse_args()
+    build(args.source_dir, args.out_dir)
+
+
+if __name__ == '__main__':
+    main()

pep517/wrappers.py

@@ -1,8 +1,9 @@
+import threading
 from contextlib import contextmanager
 import os
 from os.path import dirname, abspath, join as pjoin
 import shutil
-from subprocess import check_call
+from subprocess import check_call, check_output, STDOUT
 import sys
 from tempfile import mkdtemp
 
@@ -49,6 +50,15 @@ def default_subprocess_runner(cmd, cwd=None, extra_environ=None):
     check_call(cmd, cwd=cwd, env=env)
 
 
+def quiet_subprocess_runner(cmd, cwd=None, extra_environ=None):
+    """A method of calling the wrapper subprocess while suppressing output."""
+    env = os.environ.copy()
+    if extra_environ:
+        env.update(extra_environ)
+
+    check_output(cmd, cwd=cwd, env=env, stderr=STDOUT)
+
+
 def norm_and_check(source_tree, requested):
     """Normalise and check a backend path.
 
@@ -228,3 +238,37 @@ def _call_hook(self, hook_name, kwargs):
                     message=data.get('backend_error', '')
                 )
             return data['return_val']
+
+
+class LoggerWrapper(threading.Thread):
+    """
+    Read messages from a pipe and redirect them
+    to a logger (see python's logging module).
+    """
+
+    def __init__(self, logger, level):
+        threading.Thread.__init__(self)
+        self.daemon = True
+
+        self.logger = logger
+        self.level = level
+
+        # create the pipe and reader
+        self.fd_read, self.fd_write = os.pipe()
+        self.reader = os.fdopen(self.fd_read)
+
+        self.start()
+
+    def fileno(self):
+        return self.fd_write
+
+    @staticmethod
+    def remove_newline(msg):
+        return msg[:-1] if msg.endswith(os.linesep) else msg
+
+    def run(self):
+        for line in self.reader:
+            self._write(self.remove_newline(line))
+
+    def _write(self, message):
+        self.logger.log(self.level, message)

pyproject.toml

@@ -10,6 +10,8 @@ home-page = "https://github.com/takluyver/pep517"
 description-file = "README.rst"
 requires = [
     "pytoml",
+    "importlib_metadata",
+    "zipp",
 ]
 classifiers = ["License :: OSI Approved :: MIT License"]
 

tests/test_build.py

@@ -0,0 +1,27 @@
+import pytest
+
+from pep517 import build
+
+
+def system(*args):
+    return dict.fromkeys(args)
+
+
+class TestValidateSystem:
+    def test_missing(self):
+        with pytest.raises(ValueError):
+            build.validate_system(system())
+        with pytest.raises(ValueError):
+            build.validate_system(system('requires'))
+        with pytest.raises(ValueError):
+            build.validate_system(system('build-backend'))
+        with pytest.raises(ValueError):
+            build.validate_system(system('other'))
+
+    def test_missing_and_extra(self):
+        with pytest.raises(ValueError):
+            build.validate_system(system('build-backend', 'other'))
+
+    def test_satisfied(self):
+        build.validate_system(system('build-backend', 'requires'))
+        build.validate_system(system('build-backend', 'requires', 'other'))