Better handling of packaging env creation (#1807)

Author: gaborbernat

docs/changelog/1782.bugfix.rst

@@ -0,0 +1,2 @@
+Fixed a bug that crashed tox where calling tox with the recreate flag and when multiple environments were reusing the
+same package - by :user:`gaborbernat`.

docs/changelog/1804.bugfix.rst

@@ -0,0 +1,16 @@
+Rework how we handle python packaging environments:
+
+- the base packaging environment changed from ``.package`` to ``.pkg``,
+- merged the ``sdist``, ``wheel`` and ``dev`` separate packaging implementations into one, and internally dynamically
+  pick the one that's needed,
+- the base packaging environment always uses the same python environment as tox is installed into,
+- the base packaging environment is used to get the metadata of the project (via PEP-517) and to build ``sdist`` and
+  ``dev`` packages,
+- for building wheels introduced a new per env configurable option ``wheel_build_env``, if the target python major/minor
+  and implementation for the run tox environment and the base package tox environment matches set this to ``.pkg``,
+  otherwise this is ``.pkg-{implementation}{major}{minor}``,
+- internally now packaging environments can create further packaging environments they are responsible of managing,
+- updated ``depends`` to use the packaging logic,
+- add support skip missing interpreters for depends and show config,
+
+by :user:`gaborbernat`.

src/tox/config/sets.py

@@ -32,7 +32,7 @@ class ConfigSet:
     """A set of configuration that belong together (such as a tox environment settings, core tox settings)"""
 
     def __init__(self, conf: "Config", name: Optional[str]):
-        self.name = name
+        self._name = name
         self._conf = conf
         self.loaders: List[Loader[Any]] = []
         self._defined: Dict[str, ConfigDefinition[Any]] = {}
@@ -51,13 +51,13 @@ def add_config(
         Add configuration value.
         """
         keys_ = self._make_keys(keys)
-        definition = ConfigDynamicDefinition(keys_, desc, self.name, of_type, default, post_process)
+        definition = ConfigDynamicDefinition(keys_, desc, self._name, of_type, default, post_process)
         result = self._add_conf(keys_, definition)
         return cast(ConfigDynamicDefinition[V], result)
 
     def add_constant(self, keys: Union[str, Sequence[str]], desc: str, value: V) -> ConfigConstantDefinition[V]:
         keys_ = self._make_keys(keys)
-        definition = ConfigConstantDefinition(keys_, desc, self.name, value)
+        definition = ConfigConstantDefinition(keys_, desc, self._name, value)
         result = self._add_conf(keys_, definition)
         return cast(ConfigConstantDefinition[V], result)
 
@@ -70,7 +70,7 @@ def _add_conf(self, keys: Sequence[str], definition: ConfigDefinition[V]) -> Con
         if key in self._defined:
             earlier = self._defined[key]
             # core definitions may be defined multiple times as long as all their options match, first defined wins
-            if self.name is None and definition == earlier:
+            if self._name is None and definition == earlier:
                 definition = earlier
             else:
                 raise ValueError(f"config {key} already defined")
@@ -93,7 +93,7 @@ def load(self, item: str, chain: Optional[List[str]] = None) -> Any:
         return config_definition(self._conf, item, self.loaders, chain)
 
     def __repr__(self) -> str:
-        values = (v for v in (f"name={self.name!r}" if self.name else "", f"loaders={self.loaders!r}") if v)
+        values = (v for v in (f"name={self._name!r}" if self._name else "", f"loaders={self.loaders!r}") if v)
         return f"{self.__class__.__name__}({', '.join(values)})"
 
     def __iter__(self) -> Iterator[str]:
@@ -158,6 +158,10 @@ def set_env_post_process(values: SetEnv, config: "Config") -> SetEnv:
             post_process=set_env_post_process,
         )
 
+    @property
+    def name(self) -> str:
+        return self._name  # type: ignore
+
 
 __all__ = (
     "ConfigSet",

src/tox/plugin/manager.py

@@ -13,7 +13,7 @@
 from tox.tox_env import package as package_api
 from tox.tox_env.api import ToxEnv
 from tox.tox_env.python.virtual_env import runner
-from tox.tox_env.python.virtual_env.package.artifact import dev, sdist, wheel
+from tox.tox_env.python.virtual_env.package import api
 from tox.tox_env.register import REGISTER, ToxEnvRegister
 
 from . import NAME, spec
@@ -28,9 +28,7 @@ def __init__(self) -> None:
             loader_api,
             provision,
             runner,
-            dev,
-            sdist,
-            wheel,
+            api,
             legacy,
             version_flag,
             quickstart,

src/tox/pytest.py

@@ -193,16 +193,17 @@ def cmd(self) -> Sequence[str]:
 
         @contextmanager
         def _execute_call(
-            executor: Execute, out_err: OutErr, request: ExecuteRequest, show: bool
+            self: ToxEnv, executor: Execute, out_err: OutErr, request: ExecuteRequest, show: bool  # noqa
         ) -> Iterator[ExecuteStatus]:
             exit_code = handle(request)
             if exit_code is not None:
                 executor = MockExecute(colored=executor._colored, exit_code=exit_code)  # noqa
-            with original_execute_call(executor, out_err, request, show) as status:
+            with original_execute_call(self, executor, out_err, request, show) as status:
                 yield status
 
         original_execute_call = ToxEnv._execute_call  # noqa
-        return self.mocker.patch.object(ToxEnv, "_execute_call", side_effect=_execute_call)
+        result = self.mocker.patch.object(ToxEnv, "_execute_call", side_effect=_execute_call, autospec=True)
+        return result
 
     @property
     def structure(self) -> Dict[str, Any]:

src/tox/session/cmd/depends.py

@@ -17,20 +17,22 @@ def tox_add_option(parser: ToxParser) -> None:
 
 
 def depends(state: State) -> int:
-    to_run_list = list(state.env_list())
+    to_run_list = list(state.env_list(everything=True))
     order, todo = run_order(state, to_run_list)
     print(f"Execution order: {', '.join(order)}")
 
     deps: Dict[str, List[str]] = {k: [o for o in order if o in v] for k, v in todo.items()}
     deps["ALL"] = to_run_list
 
     def _handle(at: int, env: str) -> None:
+        if env not in order and env != "ALL":  # skipped envs
+            return
         print("   " * at, end="")
         print(env, end="")
         if env != "ALL":
-            pkg_env = state.tox_env(env).package_env
-            if pkg_env is not None:
-                print(f" ~ {pkg_env.conf['env_name']}", end="")
+            names = " | ".join(e.conf.name for e in state.tox_env(env).package_envs())
+            if names:
+                print(f" ~ {names}", end="")
         print("")
         at += 1
         for dep in deps[env]:

src/tox/session/cmd/run/common.py

@@ -17,6 +17,7 @@
 from tox.session.cmd.run.single import ToxEnvRunResult, run_one
 from tox.session.state import State
 from tox.tox_env.api import ToxEnv
+from tox.tox_env.errors import Skip
 from tox.tox_env.runner import RunToxEnv
 from tox.util.graph import stable_topological_sort
 from tox.util.spinner import MISS_DURATION, Spinner
@@ -230,7 +231,7 @@ def _queue_and_wait(
             envs_to_run_generator = ready_to_run_envs(state, to_run_list, completed)
 
             def _run(tox_env: RunToxEnv) -> ToxEnvRunResult:
-                spinner.add(cast(str, tox_env.conf.name))
+                spinner.add(tox_env.conf.name)
                 return run_one(tox_env, options.recreate, options.no_test, suspend_display=live is False)
 
             try:
@@ -256,7 +257,7 @@ def _run(tox_env: RunToxEnv) -> ToxEnvRunResult:
                         result: ToxEnvRunResult = future.result()
                     except CancelledError:
                         tox_env_done.teardown()
-                        name = cast(str, tox_env_done.conf.name)
+                        name = tox_env_done.conf.name
                         result = ToxEnvRunResult(name=name, skipped=False, code=-3, outcomes=[], duration=MISS_DURATION)
                     results.append(result)
                     completed.add(result.name)
@@ -304,8 +305,13 @@ def ready_to_run_envs(state: State, to_run: List[str], completed: Set[str]) -> I
 
 def run_order(state: State, to_run: List[str]) -> Tuple[List[str], Dict[str, Set[str]]]:
     to_run_set = set(to_run)
-    todo: Dict[str, Set[str]] = {
-        env: (to_run_set & set(cast(EnvList, state.tox_env(env).conf["depends"]).envs)) for env in to_run
-    }
+    todo: Dict[str, Set[str]] = {}
+    for env in to_run:
+        try:
+            run_env = state.tox_env(env)
+        except Skip:
+            continue
+        depends = set(cast(EnvList, run_env.conf["depends"]).envs)
+        todo[env] = to_run_set & depends
     order = stable_topological_sort(todo)
     return order, todo

src/tox/session/cmd/run/single.py

@@ -25,7 +25,7 @@ class ToxEnvRunResult(NamedTuple):
 
 def run_one(tox_env: RunToxEnv, recreate: bool, no_test: bool, suspend_display: bool) -> ToxEnvRunResult:
     start_one = time.monotonic()
-    name = cast(str, tox_env.conf.name)
+    name = tox_env.conf.name
     with tox_env.display_context(suspend_display):
         skipped, code, outcomes = _evaluate(tox_env, recreate, no_test)
     duration = time.monotonic() - start_one

src/tox/session/cmd/show_config.py

@@ -3,14 +3,16 @@
 """
 
 from textwrap import indent
-from typing import Iterable, List
+from typing import Iterable, List, Set
 
 from tox.config.cli.parser import ToxParser
 from tox.config.loader.stringify import stringify
 from tox.config.sets import ConfigSet
 from tox.plugin.impl import impl
 from tox.session.common import env_list_flag
 from tox.session.state import State
+from tox.tox_env.api import ToxEnv
+from tox.tox_env.errors import Skip
 
 
 @impl
@@ -27,20 +29,40 @@ def tox_add_option(parser: ToxParser) -> None:
 
 
 def show_config(state: State) -> int:
-    show_core = state.options.env.all or state.options.show_core
     keys: List[str] = state.options.list_keys_only
-    # environments may define core configuration flags, so we must exhaust first the environments to tell the core part
-    envs = list(state.env_list(everything=False))
-    for at, name in enumerate(envs):
-        tox_env = state.tox_env(name)
-        print(f"[testenv:{name}]")
+    is_first = True
+    selected = state.options.env
+
+    def _print_env(tox_env: ToxEnv) -> None:
+        nonlocal is_first
+        if is_first:
+            is_first = False
+        else:
+            print("")
+        print(f"[testenv:{tox_env.conf.name}]")
         if not keys:
             print(f"type = {type(tox_env).__name__}")
         print_conf(tox_env.conf, keys)
-        if show_core or at + 1 != len(envs):
-            print("")
-    # no print core
-    if show_core:
+
+    envs = list(state.env_list(everything=True))
+    done_pkg_envs: Set[str] = set()
+    for name in envs:
+        try:
+            run_env = state.tox_env(name)
+        except Skip:
+            run_env = state.tox_env(name)  # get again to get the temporary state
+        if run_env.conf.name in selected:
+            _print_env(run_env)
+        for pkg_env in run_env.package_envs():
+            if pkg_env.conf.name in done_pkg_envs:
+                continue
+            done_pkg_envs.add(pkg_env.conf.name)
+            if pkg_env.conf.name in selected:
+                _print_env(pkg_env)
+
+    # environments may define core configuration flags, so we must exhaust first the environments to tell the core part
+    if selected.all or state.options.show_core:
+        print("")
         print("[tox]")
         print_conf(state.conf.core, keys)
     return 0

src/tox/session/common.py

@@ -27,6 +27,9 @@ def __eq__(self, other: Any) -> bool:
     def __ne__(self, other: Any) -> bool:
         return not (self == other)
 
+    def __contains__(self, item: str) -> bool:
+        return self.all or (self._names is not None and item in self._names)
+
 
 def env_list_flag(parser: ArgumentParser, default: Optional[CliEnv] = None) -> None:
     parser.add_argument(

src/tox/session/state.py

@@ -32,7 +32,7 @@ def __init__(
         self.args = args
 
         self._run_env: Dict[str, RunToxEnv] = {}
-        self._pkg_env: Dict[str, PackageToxEnv] = {}
+        self._pkg_env: Dict[str, Tuple[str, PackageToxEnv]] = {}
         self._pkg_env_discovered: Set[str] = set()
 
         self.journal: Journal = Journal(getattr(options, "result_json", None) is not None)
@@ -86,29 +86,33 @@ def _build_run_env(self, env_conf: EnvConfigSet) -> None:
         from tox.tox_env.register import REGISTER
 
         builder = REGISTER.runner(runner)
-        name = cast(str, env_conf.name)
+        name = env_conf.name
         journal = self.journal.get_env_journal(name)
         env: RunToxEnv = builder(env_conf, self.conf.core, self.options, journal, self.log_handler)
         self._run_env[name] = env
         self._build_package_env(env)
 
     def _build_package_env(self, env: RunToxEnv) -> None:
-        pkg_env_gen = env.set_package_env()
-        try:
-            name, packager = next(pkg_env_gen)
-        except StopIteration:
-            pass
-        else:
-            with self.log_handler.with_context(name):
-                package_tox_env = self._get_package_env(packager, name)
-                try:
-                    pkg_env_gen.send(package_tox_env)
-                except StopIteration:
-                    pass
+        pkg_env_gen = env.create_package_env()
+        while True:
+            try:
+                name, packager = next(pkg_env_gen)
+            except StopIteration:
+                return
+            else:
+                with self.log_handler.with_context(name):
+                    package_tox_env = self._get_package_env(packager, name)
+                    try:
+                        pkg_env_gen.send(package_tox_env)
+                    except StopIteration:
+                        return
 
     def _get_package_env(self, packager: str, name: str) -> PackageToxEnv:
         if name in self._pkg_env:  # if already created reuse
-            pkg_tox_env: PackageToxEnv = self._pkg_env[name]
+            old, pkg_tox_env = self._pkg_env[name]
+            if old != packager:  # pragma: no branch # same env name is used by different packaging: dpkg vs virtualenv
+                msg = f"{name} is already defined as a {old}, cannot be {packager} too"  # pragma: no cover
+                raise HandledError(msg)  # pragma: no cover
         else:
             from tox.tox_env.register import REGISTER
 
@@ -119,7 +123,7 @@ def _get_package_env(self, packager: str, name: str) -> PackageToxEnv:
             pkg_conf = self.conf.get_env(name, package=True)
             journal = self.journal.get_env_journal(name)
             pkg_tox_env = package_type(pkg_conf, self.conf.core, self.options, journal, self.log_handler)
-            self._pkg_env[name] = pkg_tox_env
+            self._pkg_env[name] = packager, pkg_tox_env
         return pkg_tox_env
 
 

src/tox/tox_env/api.py

@@ -49,6 +49,7 @@ def __init__(
         self.clean_done = False
         self._execute_statuses: Dict[int, ExecuteStatus] = {}
         self._interrupted = False
+        self.skipped = False
 
     def interrupt(self) -> None:
         logging.warning("interrupt tox environment: %s", self.conf.name)
@@ -153,7 +154,7 @@ def setup(self) -> None:
                 if eq is False and old is not None:  # recreate if already created and not equals
                     logging.warning(f"env type changed from {old} to {conf}, will recreate")
                     raise Recreate  # recreate if already exists and type changed
-                self.setup_done, self.clean_done = True, False
+                self.setup_done = True
         finally:
             self._handle_env_tmp_dir()
 
@@ -270,10 +271,9 @@ def execute_async(
         if self.journal and execute_status.outcome is not None:
             self.journal.add_execute(execute_status.outcome, run_id)
 
-    @staticmethod
     @contextmanager
     def _execute_call(
-        executor: Execute, out_err: OutErr, request: ExecuteRequest, show: bool
+        self, executor: Execute, out_err: OutErr, request: ExecuteRequest, show: bool
     ) -> Iterator[ExecuteStatus]:
         with executor.call(
             request=request,
@@ -306,7 +306,7 @@ def close_and_read_out_err(self) -> Optional[Tuple[bytes, bytes]]:
 
     @contextmanager
     def log_context(self) -> Iterator[None]:
-        with self.log_handler.with_context(cast(str, self.conf.name)):
+        with self.log_handler.with_context(self.conf.name):
             yield
 
     def teardown(self) -> None:

src/tox/tox_env/info.py

@@ -47,6 +47,7 @@ def reset(self) -> None:
         self._content = {}
 
     def _write(self) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
         self._path.write_text(json.dumps(self._content, indent=2))