diff --git a/news/7962.bugfix b/news/7962.bugfix new file mode 100644 index 00000000000..76c3442d053 --- /dev/null +++ b/news/7962.bugfix @@ -0,0 +1 @@ +Significantly speedup ``pip list --outdated`` through parallelizing index interaction. diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 109ec5c664e..cf3be7eb459 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -5,8 +5,10 @@ import json import logging +from multiprocessing.dummy import Pool from pip._vendor import six +from pip._vendor.requests.adapters import DEFAULT_POOLSIZE from pip._internal.cli import cmdoptions from pip._internal.cli.req_command import IndexGroupCommand @@ -183,7 +185,7 @@ def iter_packages_latest_infos(self, packages, options): with self._build_session(options) as session: finder = self._build_package_finder(options, session) - for dist in packages: + def latest_info(dist): typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) if not options.pre: @@ -196,7 +198,7 @@ def iter_packages_latest_infos(self, packages, options): ) best_candidate = evaluator.sort_best_candidate(all_candidates) if best_candidate is None: - continue + return None remote_version = best_candidate.version if best_candidate.link.is_wheel: @@ -206,7 +208,19 @@ def iter_packages_latest_infos(self, packages, options): # This is dirty but makes the rest of the code much cleaner dist.latest_version = remote_version dist.latest_filetype = typ - yield dist + return dist + + # This is done for 2x speed up of requests to pypi.org + # so that "real time" of this function + # is almost equal to "user time" + pool = Pool(DEFAULT_POOLSIZE) + + for dist in pool.imap_unordered(latest_info, packages): + if dist is not None: + yield dist + + pool.close() + pool.join() def output_package_listing(self, packages, options): packages = sorted( diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 134f7908d9d..9a017cf7e33 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -52,7 +52,6 @@ _log_state = threading.local() -_log_state.indentation = 0 subprocess_logger = getLogger('pip.subprocessor') @@ -104,6 +103,8 @@ def indent_log(num=2): A context manager which will cause the log output to be indented for any log messages emitted inside it. """ + # For thread-safety + _log_state.indentation = get_indentation() _log_state.indentation += num try: yield diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index a2bab3ea9c5..a62c18c770f 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -2,6 +2,7 @@ import logging import os import time +from threading import Thread import pytest from mock import patch @@ -11,6 +12,7 @@ BrokenStdoutLoggingError, ColorizedStreamHandler, IndentingFormatter, + indent_log, ) from pip._internal.utils.misc import captured_stderr, captured_stdout @@ -108,6 +110,39 @@ def test_format_deprecated(self, level_name, expected): f = IndentingFormatter(fmt="%(message)s") assert f.format(record) == expected + def test_thread_safety_base(self): + record = self.make_record( + 'DEPRECATION: hello\nworld', level_name='WARNING', + ) + f = IndentingFormatter(fmt="%(message)s") + results = [] + + def thread_function(): + results.append(f.format(record)) + + thread_function() + thread = Thread(target=thread_function) + thread.start() + thread.join() + assert results[0] == results[1] + + def test_thread_safety_indent_log(self): + record = self.make_record( + 'DEPRECATION: hello\nworld', level_name='WARNING', + ) + f = IndentingFormatter(fmt="%(message)s") + results = [] + + def thread_function(): + with indent_log(): + results.append(f.format(record)) + + thread_function() + thread = Thread(target=thread_function) + thread.start() + thread.join() + assert results[0] == results[1] + class TestColorizedStreamHandler(object):