diff --git a/docs/reference/index.rst b/docs/reference/index.rst index cb83554b30e..261934935ea 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -15,3 +15,4 @@ Reference Guide pip_search pip_wheel pip_hash + pip_run diff --git a/docs/reference/pip_run.rst b/docs/reference/pip_run.rst new file mode 100644 index 00000000000..f7ce31ec956 --- /dev/null +++ b/docs/reference/pip_run.rst @@ -0,0 +1,36 @@ +pip run +------- + +.. contents:: + +Usage +***** + +.. pip-command-usage:: run + +Description +*********** + +.. pip-command-description:: run + + +Overview +++++++++ + +Pip run in a specialized invocation of pip install that makes +packages available only for the duration of a single Python invocation +for one-off needs. The command is based on the +`rwt project `_. + +Argument Handling ++++++++++++++++++ + +As a wrapper around ``pip install``, the arguments to ``run`` are split into +two segments, separated by a double-dash (``--``). The arguments prior +to the ``--`` are passed directly to :ref:`pip install`, so should contain +requirements, requirments files, and index directives. + +The arguments after the ``--`` are passed to a new Python interpreter in the +context of the installed dependencies. + +For more details and examples, see the `rwt project `_. diff --git a/pip/_vendor/Makefile b/pip/_vendor/Makefile index 175e4166768..3f6834af5ae 100644 --- a/pip/_vendor/Makefile +++ b/pip/_vendor/Makefile @@ -8,6 +8,9 @@ vendor: @# Install vendored libraries pip install -t . -r vendor.txt + # remove the tests + rm -rf rwt/tests + @# Cleanup .egg-info directories rm -rf *.egg-info rm -rf *.dist-info diff --git a/pip/_vendor/__init__.py b/pip/_vendor/__init__.py index bee5f5e6fd3..d90cdeb6371 100644 --- a/pip/_vendor/__init__.py +++ b/pip/_vendor/__init__.py @@ -105,3 +105,4 @@ def vendored(modulename): vendored("requests.packages.urllib3.util.ssl_") vendored("requests.packages.urllib3.util.timeout") vendored("requests.packages.urllib3.util.url") + vendored("rwt") diff --git a/pip/_vendor/re-vendor.py b/pip/_vendor/re-vendor.py index 0a52123e4f2..16c1165edf4 100644 --- a/pip/_vendor/re-vendor.py +++ b/pip/_vendor/re-vendor.py @@ -19,7 +19,7 @@ def clean(): os.unlink(os.path.join(here, 'six.py')) def vendor(): - pip.main(['install', '-t', here, '-r', 'vendor.txt']) + pip.main(['install', '-t', here, '-r', os.path.join(here, 'vendor.txt')]) for dirname in glob.glob('*.egg-info'): shutil.rmtree(dirname) diff --git a/pip/_vendor/rwt/__init__.py b/pip/_vendor/rwt/__init__.py new file mode 100644 index 00000000000..0d3be9e58e4 --- /dev/null +++ b/pip/_vendor/rwt/__init__.py @@ -0,0 +1,16 @@ +import sys + +from . import deps +from . import commands +from . import launch +from . import scripts + + +def run(args=None): + if args is None: + args = sys.argv[1:] + pip_args, params = commands.parse_script_args(args) + commands.intercept(pip_args) + pip_args.extend(scripts.DepsReader.search(params)) + with deps.load(*pip_args) as home: + launch.with_path(home, params) diff --git a/pip/_vendor/rwt/__main__.py b/pip/_vendor/rwt/__main__.py new file mode 100644 index 00000000000..ff288dd4f96 --- /dev/null +++ b/pip/_vendor/rwt/__main__.py @@ -0,0 +1,4 @@ +from . import run + + +__name__ == '__main__' and run() diff --git a/pip/_vendor/rwt/commands.py b/pip/_vendor/rwt/commands.py new file mode 100644 index 00000000000..03ba78acf82 --- /dev/null +++ b/pip/_vendor/rwt/commands.py @@ -0,0 +1,55 @@ +import textwrap + + +def parse_script_args(args): + """ + Separate the command line arguments into arguments for pip + and arguments to Python. + + >>> parse_script_args(['foo', '--', 'bar']) + (['foo'], ['bar']) + + >>> parse_script_args(['foo', 'bar']) + (['foo', 'bar'], []) + """ + try: + pivot = args.index('--') + except ValueError: + pivot = len(args) + return args[:pivot], args[pivot+1:] + + +help_doc = textwrap.dedent(""" + Usage: + + Arguments to rwt prior to `--` are used to specify the requirements + to make available, just as arguments to pip install. For example, + + rwt -r requirements.txt "requests>=2.0" + + That will launch python after installing the deps in requirements.txt + and also a late requests. Packages are always installed to a temporary + location and cleaned up when the process exits. + + Arguments after `--` are passed to the Python interpreter. So to launch + `script.py`: + + rwt -- script.py + + If the `--` is ommitted or nothing is passed, the python interpreter + will be launched in interactive mode: + + rwt + >>> + + For more examples and details, see https://pypi.org/project/rwt. + """).lstrip() + + +def intercept(args): + """ + Detect certain args and intercept them. + """ + if '--help' in args or '-h' in args: + print(help_doc) + raise SystemExit(0) diff --git a/pip/_vendor/rwt/deps.py b/pip/_vendor/rwt/deps.py new file mode 100644 index 00000000000..82422ce7a55 --- /dev/null +++ b/pip/_vendor/rwt/deps.py @@ -0,0 +1,95 @@ +from __future__ import print_function + +import sys +import contextlib +import subprocess +import tempfile +import shutil + + +@contextlib.contextmanager +def _update_working_set(): + """ + Update the master working_set to include these new packages. + + TODO: would be better to use an officially-supported API, + but no suitable API is apparent. + """ + try: + pkg_resources = sys.modules['pkg_resources'] + if not hasattr(pkg_resources, '_initialize_master_working_set'): + exec(_init_ws_patch, vars(pkg_resources)) + pkg_resources._initialize_master_working_set() + except KeyError: + # it's unnecessary to re-initialize when it hasn't + # yet been initialized. + pass + yield + + +@contextlib.contextmanager +def load(*args): + target = tempfile.mkdtemp(prefix='rwt-') + cmdline = subprocess.list2cmdline(args) + print("Loading requirements using", cmdline) + cmd = ( + sys.executable, + '-m', 'pip', + 'install', + '-q', + '-t', target, + ) + args + subprocess.check_call(cmd) + try: + yield target + finally: + shutil.rmtree(target) + + +@contextlib.contextmanager +def on_sys_path(*args): + """ + Install dependencies via args to pip and ensure they have precedence + on sys.path. + """ + with load(*args) as target: + sys.path.insert(0, target) + try: + with _update_working_set(): + yield target + finally: + sys.path.remove(target) + + +# from setuptools 19.6.2 +_init_ws_patch = ''' +def _initialize_master_working_set(): + """ + Prepare the master working set and make the ``require()`` + API available. + + This function has explicit effects on the global state + of pkg_resources. It is intended to be invoked once at + the initialization of this module. + + Invocation by other packages is unsupported and done + at their own risk. + """ + working_set = WorkingSet._build_master() + _declare_state('object', working_set=working_set) + + require = working_set.require + iter_entry_points = working_set.iter_entry_points + add_activation_listener = working_set.subscribe + run_script = working_set.run_script + # backward compatibility + run_main = run_script + # Activate all distributions already on sys.path, and ensure that + # all distributions added to the working set in the future (e.g. by + # calling ``require()``) will get activated as well. + add_activation_listener(lambda dist: dist.activate()) + working_set.entries=[] + # match order + list(map(working_set.add_entry, sys.path)) + globals().update(locals()) +''' diff --git a/pip/_vendor/rwt/launch.py b/pip/_vendor/rwt/launch.py new file mode 100644 index 00000000000..8ff77ddf892 --- /dev/null +++ b/pip/_vendor/rwt/launch.py @@ -0,0 +1,91 @@ +import textwrap +import os +import subprocess +import sys +import signal +import glob +import itertools + + +class PathReader: + @staticmethod + def _read_file(filename): + root = os.path.dirname(filename) + return ( + os.path.join(root, path.rstrip()) + for path in open(filename) + if path.strip() + and not path.startswith('#') + and not path.startswith('import ') + ) + + @classmethod + def _read(cls, target): + """ + As .pth files aren't honored except in site dirs, + read the paths indicated by them. + """ + pth_files = glob.glob(os.path.join(target, '*.pth')) + file_items = map(cls._read_file, pth_files) + return itertools.chain.from_iterable(file_items) + + +def _inject_sitecustomize(target): + """ + Create a sitecustomize file in the target that will install + the target as a sitedir. + + Only needed on Python 3.2 and earlier to workaround #1. + """ + if sys.version_info > (3, 3): + return + + hook = textwrap.dedent(""" + import site + site.addsitedir({target!r}) + """).lstrip().format(**locals()) + sc_fn = os.path.join(target, 'sitecustomize.py') + with open(sc_fn, 'w') as strm: + strm.write(hook) + + +def _build_env(target): + """ + Prepend target and .pth references in target to PYTHONPATH + """ + env = dict(os.environ) + suffix = env.get('PYTHONPATH') + prefix = target, + items = itertools.chain( + prefix, + PathReader._read(target), + (suffix,) if suffix else (), + ) + joined = os.pathsep.join(items) + env['PYTHONPATH'] = joined + return env + + +def _setup_env(target): + _inject_sitecustomize(target) + return _build_env(target) + + +def with_path(target, params): + """ + Launch Python with target on the path and params + """ + def null_handler(signum, frame): + pass + + signal.signal(signal.SIGINT, null_handler) + cmd = [sys.executable] + params + subprocess.Popen(cmd, env=_setup_env(target)).wait() + + +def with_path_overlay(target, params): + """ + Overlay Python with target on the path and params + """ + cmd = [sys.executable] + params + os.execve(sys.executable, cmd, _setup_env(target)) diff --git a/pip/_vendor/rwt/scripts.py b/pip/_vendor/rwt/scripts.py new file mode 100644 index 00000000000..8abd19a7e9f --- /dev/null +++ b/pip/_vendor/rwt/scripts.py @@ -0,0 +1,81 @@ +import os +import sys +import ast +import tokenize +import itertools + + +if sys.version_info < (3,): + filter = itertools.ifilter + + +class DepsReader: + """ + Given a Python script, read the dependencies from the + indicated variable (default __requires__). Does not + execute the script, so expects the var_name to be + assigned a static list of strings. + """ + def __init__(self, script): + self.script = script + + @classmethod + def load(cls, script_path): + with open(script_path) as stream: + return cls(stream.read()) + + @classmethod + def try_read(cls, script_path): + """ + Attempt to load the dependencies from the script, + but return an empty list if unsuccessful. + """ + try: + reader = cls.load(script_path) + return reader.read() + except Exception: + return [] + + @classmethod + def search(cls, params): + """ + Given a (possibly-empty) series of parameters to a + Python interpreter, return any dependencies discovered + in a script indicated in the parameters. Only honor the + first file found. + """ + files = filter(os.path.isfile, params) + return cls.try_read(next(files, None)) + + def read(self, var_name='__requires__'): + """ + >>> DepsReader("__requires__=['foo']").read() + ['foo'] + """ + mod = ast.parse(self.script) + node, = ( + node + for node in mod.body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == var_name + ) + return ast.literal_eval(node.value) + + +def run(cmdline): + """ + Execute the script as if it had been invoked naturally. + """ + namespace = dict() + filename = cmdline[0] + namespace['__file__'] = filename + namespace['__name__'] = '__main__' + sys.argv[:] = cmdline + + open_ = getattr(tokenize, 'open', open) + script = open_(filename).read() + norm_script = script.replace('\\r\\n', '\\n') + code = compile(norm_script, filename, 'exec') + exec(code, namespace) diff --git a/pip/_vendor/vendor.txt b/pip/_vendor/vendor.txt index f82b97d1390..1e84ec0dad4 100644 --- a/pip/_vendor/vendor.txt +++ b/pip/_vendor/vendor.txt @@ -11,3 +11,4 @@ ipaddress==1.0.16 # Only needed on 2.6 and 2.7 packaging==16.7 pyparsing==2.1.1 retrying==1.3.3 +rwt==2.12 diff --git a/pip/commands/__init__.py b/pip/commands/__init__.py index 62c64ebed27..3db3710288a 100644 --- a/pip/commands/__init__.py +++ b/pip/commands/__init__.py @@ -15,6 +15,7 @@ from pip.commands.install import InstallCommand from pip.commands.uninstall import UninstallCommand from pip.commands.wheel import WheelCommand +from pip.commands.run import RunCommand commands_dict = { @@ -30,6 +31,7 @@ ListCommand.name: ListCommand, CheckCommand.name: CheckCommand, WheelCommand.name: WheelCommand, + RunCommand.name: RunCommand, } @@ -45,6 +47,7 @@ WheelCommand, HashCommand, CompletionCommand, + RunCommand, HelpCommand, ] diff --git a/pip/commands/run.py b/pip/commands/run.py new file mode 100644 index 00000000000..575d32ea17f --- /dev/null +++ b/pip/commands/run.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import + +from pip.basecommand import Command, SUCCESS + +from pip._vendor import rwt + + +class RunCommand(Command): + """Run a new Python interpreter with packages transient-installed""" + name = 'run' + usage = rwt.commands.help_doc + summary = 'Run Python with dependencies loaded.' + + def main(self, args): + if ['--help'] == args: + return super(RunCommand, self).main(args) + + rwt.run(args) + + return SUCCESS