diff --git a/docs/source/_static/images/markers.svg b/docs/source/_static/images/markers.svg index f27a5e7d..5febd4cd 100644 --- a/docs/source/_static/images/markers.svg +++ b/docs/source/_static/images/markers.svg @@ -107,9 +107,6 @@
│ pytask.mark.depends_on │ Add dependencies to a task. See this tutorial for more │
│ │ information: https://bit.ly/3JlxylS. │
│ │ │
-
│ pytask.mark.parametrize │ The marker for pytest's way of repeating tasks which is │
-
│ │ explained in this tutorial: https://bit.ly/3uqZqkk. │
-
│ │ │
│ pytask.mark.persist │ Prevent execution of a task if all products exist and even if │
│ │ something has changed (dependencies, source file, products). │
│ │ This decorator might be useful for expensive tasks where only │
diff --git a/docs/source/_static/md/markers.md b/docs/source/_static/md/markers.md index 60b3d0df..2ce8c7d7 100644 --- a/docs/source/_static/md/markers.md +++ b/docs/source/_static/md/markers.md @@ -10,10 +10,6 @@ $ pytask markers │ │ tutorial for more information: │ │ │ https://bit.ly/3JlxylS. │ │ │ │ -│ pytask.mark.parametrize │ The marker for pytest's way of │ -│ │ repeating tasks which is explained in │ -│ │ this tutorial: https://bit.ly/3uqZqkk. │ -│ │ │ │ pytask.mark.persist │ Prevent execution of a task if all │ │ │ products exist and even ifsomething has │ │ │ changed (dependencies, source file, │ diff --git a/docs/source/changes.md b/docs/source/changes.md index 21617804..d7fa682d 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -9,6 +9,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`323` remove Python 3.7 support and use a new Github action to provide mamba. - {pull}`387` replaces pony with sqlalchemy. +- {pull}`391` removes `@pytask.mark.parametrize`. ## 0.3.2 - 2023-06-07 diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md index 29830859..7f3c9aa5 100644 --- a/docs/source/how_to_guides/index.md +++ b/docs/source/how_to_guides/index.md @@ -14,7 +14,6 @@ maxdepth: 1 migrating_from_scripts_to_pytask invoking_pytask_extended capture_warnings -repeating_tasks_with_different_inputs_the_pytest_way how_to_influence_build_order how_to_write_a_plugin ``` diff --git a/docs/source/how_to_guides/repeating_tasks_with_different_inputs_the_pytest_way.md b/docs/source/how_to_guides/repeating_tasks_with_different_inputs_the_pytest_way.md deleted file mode 100644 index 7a81c571..00000000 --- a/docs/source/how_to_guides/repeating_tasks_with_different_inputs_the_pytest_way.md +++ /dev/null @@ -1,213 +0,0 @@ -# Repeating tasks with different inputs - The pytest way - -:::{important} -This guide shows you how to parametrize tasks with the pytest approach. For the new and -preferred approach, see this -{doc}`tutorial <../tutorials/repeating_tasks_with_different_inputs>`. -::: - -Do you want to define a task repeating an action over a range of inputs? Parametrize -your task function! - -:::{hint} -The process of repeating a function with different inputs is called parametrizations. -::: - -:::{seealso} -If you want to know more about best practices for parametrizations, check out this -{doc}`guide <../how_to_guides/bp_scalable_repetitions_of_tasks>` after you have made -yourself familiar with this tutorial. -::: - -## An example - -We reuse the previous example of a task that generates random data and repeat the same -operation over some seeds to receive multiple, reproducible samples. - -First, we write the task for one seed. - -```python -import numpy as np -import pytask - - -@pytask.mark.produces(BLD / "data_0.pkl") -def task_create_random_data(produces): - rng = np.random.default_rng(0) - ... -``` - -In the next step, we repeat the same task over the numbers 0, 1, and 2 and pass them to -the `seed` argument. We also vary the name of the produced file in every iteration. - -```python -@pytask.mark.parametrize( - "produces, seed", - [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], -) -def task_create_random_data(seed, produces): - rng = np.random.default_rng(seed) - ... -``` - -The parametrize decorator receives two arguments. The first argument is -`"produces, seed"` - the signature. It is a comma-separated string where each value -specifies the name of a task function argument. - -:::{seealso} -The signature is explained in detail {ref}`below `. -::: - -The second argument of the parametrize decorator is a list with one element per -iteration. Each element must provide one value for each argument name in the signature - -two in this case. - -pytask executes the task function three times and passes the path from the list to the -argument `produces` and the seed to `seed`. - -:::{note} -If you use `produces` or `depends_on` in the signature of the parametrize decorator, the -values are handled as if they were attached to the function with -{func}`@pytask.mark.depends_on ` or -{func}`@pytask.mark.produces `. -::: - -## Un-parametrized dependencies - -To specify a dependency that is the same for all parametrizations, add it with -{func}`@pytask.mark.depends_on `. - -```python -@pytask.mark.depends_on(SRC / "common_dependency.file") -@pytask.mark.parametrize( - "produces, seed", - [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], -) -def task_create_random_data(seed, produces): - rng = np.random.default_rng(seed) - ... -``` - -(parametrize-signature)= - -## The signature - -pytask allows for three different kinds of formats for the signature. - -1. The signature can be a comma-separated string like an entry in a CSV table. Note that - white space is stripped from each name which you can use to separate the names for - readability. Here are some examples: - - ```python - "single_argument" - "first_argument,second_argument" - "first_argument, second_argument" - ``` - -1. The signature can be a tuple of strings where each string is one argument name. Here - is an example. - - ```python - ("first_argument", "second_argument") - ``` - -1. Finally, using a list of strings is also possible. - - ```python - ["first_argument", "second_argument"] - ``` - -## The id - -Every task has a unique id that can be used to -{doc}`select it <../tutorials/selecting_tasks>`. The normal id combines the path to the -module where the task is defined, a double colon, and the name of the task function. -Here is an example. - -``` -../task_example.py::task_example -``` - -This behavior would produce duplicate ids for parametrized tasks. Therefore, there exist -multiple mechanisms to have unique ids. - -(auto-generated-ids)= - -### Auto-generated ids - -pytask construct ids by extending the task name with representations of the values used -for each iteration. Booleans, floats, integers, and strings enter the task id directly. -For example, a task function that receives four arguments, `True`, `1.0`, `2`, and -`"hello"`, one of each dtype, has the following id. - -``` -task_example.py::task_example[True-1.0-2-hello] -``` - -Arguments with other dtypes cannot be converted to strings and, thus, are replaced with -a combination of the argument name and the iteration counter. - -For example, the following function is parametrized with tuples. - -```python -@pytask.mark.parametrize("i", [(0,), (1,)]) -def task_example(i): - pass -``` - -Since the tuples are not converted to strings, the ids of the two tasks are - -``` -task_example.py::task_example[i0] -task_example.py::task_example[i1] -``` - -### User-defined ids - -Instead of a function, you can also pass a list or another iterable of id values via -`ids`. - -This code - -```python -@pytask.mark.parametrize("i", [(0,), (1,)], ids=["first", "second"]) -def task_example(i): - pass -``` - -produces these ids - -``` -task_example.py::task_example[first] # (0,) -task_example.py::task_example[second] # (1,) -``` - -(how-to-parametrize-a-task-convert-other-objects)= - -### Convert other objects - -To change the representation of tuples and other objects, you can pass a function to the -`ids` argument of the {func}`@pytask.mark.parametrize ` -decorator. The function is called for every argument and may return a boolean, number, -or string, which will be integrated into the id. For every other return, the -auto-generated value is used. - -We can use the hash value to get a unique representation of a tuple. - -```python -def tuple_to_hash(value): - if isinstance(value, tuple): - return hash(a) - - -@pytask.mark.parametrize("i", [(0,), (1,)], ids=tuple_to_hash) -def task_example(i): - pass -``` - -The tasks have the following ids: - -``` -task_example.py::task_example[3430018387555] # (0,) -task_example.py::task_example[3430019387558] # (1,) -``` diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index 6254656b..8924bb13 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -47,35 +47,6 @@ by the host or by plugins. The following marks are available by default. :func:`_pytask.hookspecs.pytask_collect_node` entry-point. ``` -```{eval-rst} -.. function:: pytask.mark.parametrize(arg_names, arg_values, *, ids) - - Parametrize a task function. - - Parametrizing a task allows to execute the same task with different arguments. - - :type arg_names: str | list[str] | tuple[str, ...] - :param arg_names: - The names of the arguments which can either be given as a comma-separated - string, a tuple of strings, or a list of strings. - :type arg_values: Iterable[Sequence[Any] | Any] - :param arg_values: - The values which correspond to names in ``arg_names``. For one argument, it is a - single iterable. For multiple argument names it is an iterable of iterables. - :type ids: None | (Iterable[None | str | float | int | bool] | Callable[..., Any]) - :param ids: - This argument can either be a list with ids or a function which is called with - every value passed to the parametrized function. - - If you pass an iterable with ids, make sure to only use :obj:`bool`, - :obj:`float`, :obj:`int`, or :obj:`str` as values which are used to create task - ids like ``"task_dummpy.py::task_dummy[first_task_id]"``. - - If you pass a function, the function receives each value of the parametrization - and may return a boolean, number, string or None. For the latter, the - auto-generated value is used. -``` - ```{eval-rst} .. function:: pytask.mark.persist() diff --git a/docs/source/reference_guides/hookspecs.md b/docs/source/reference_guides/hookspecs.md index 474afcb4..20e958e5 100644 --- a/docs/source/reference_guides/hookspecs.md +++ b/docs/source/reference_guides/hookspecs.md @@ -106,21 +106,6 @@ The following hooks traverse directories and collect tasks from files. ``` -## Parametrization - -The hooks to parametrize a task are called during the collection when a function is -collected. Then, the function is duplicated according to the parametrization and the -duplicates are collected with {func}`pytask_collect_task`. - -```{eval-rst} -.. autofunction:: pytask_parametrize_task -``` - -```{eval-rst} -.. autofunction:: pytask_parametrize_kwarg_to_marker - -``` - ## Resolving Dependencies The following hooks are designed to build a DAG from tasks and dependencies and check diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md index 760fc37d..9b3c12f3 100644 --- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md +++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md @@ -2,16 +2,6 @@ Do you want to repeat a task over a range of inputs? Loop over your task function! -:::{important} -Before v0.2.0, pytask supported only one approach to repeat tasks. It is also called -parametrizations, and similarly to pytest, it uses a -{func}`@pytask.mark.parametrize ` decorator. If you want to -know more about it, you can find it -{doc}`here <../how_to_guides/repeating_tasks_with_different_inputs_the_pytest_way>`. - -Here you find the new and preferred approach. -::: - ## An example We reuse the task from the previous {doc}`tutorial `, which generates @@ -71,9 +61,40 @@ and the name of the task function. Here is an example. ``` This behavior would produce duplicate ids for parametrized tasks. By default, -auto-generated ids are used which are explained {ref}`here `. +auto-generated ids are used. + +(auto-generated-ids)= + +### Auto-generated ids + +pytask construct ids by extending the task name with representations of the values used +for each iteration. Booleans, floats, integers, and strings enter the task id directly. +For example, a task function that receives four arguments, `True`, `1.0`, `2`, and +`"hello"`, one of each dtype, has the following id. + +``` +task_data_preparation.py::task_create_random_data[True-1.0-2-hello] +``` + +Arguments with other dtypes cannot be converted to strings and, thus, are replaced with +a combination of the argument name and the iteration counter. -More powerful are user-defined ids. +For example, the following function is parametrized with tuples. + +```python +for i in [(0,), (1,)]: + + @pytask.mark.task + def task_create_random_data(i=i): + pass +``` + +Since the tuples are not converted to strings, the ids of the two tasks are + +``` +task_data_preparation.py::task_create_random_data[i0] +task_data_preparation.py::task_create_random_data[i1] +``` (ids)= diff --git a/pyproject.toml b/pyproject.toml index 8dc8092d..d38fcd9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,9 @@ extend-ignore = [ "SLF001", # access private members. "S603", "S607", + # Temporary + "TD002", + "TD003", ] diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index 7d42e355..4b6dd39e 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -66,7 +66,6 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: from _pytask import mark from _pytask import nodes from _pytask import parameters - from _pytask import parametrize from _pytask import persist from _pytask import profile from _pytask import dag @@ -89,7 +88,6 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: pm.register(mark) pm.register(nodes) pm.register(parameters) - pm.register(parametrize) pm.register(persist) pm.register(profile) pm.register(dag) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 7fd1dff9..dcb22df3 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -6,7 +6,6 @@ import os import sys import time -import warnings from pathlib import Path from typing import Any from typing import Generator @@ -106,15 +105,6 @@ def pytask_collect_file_protocol( return flat_reports -_PARAMETRIZE_DEPRECATION_WARNING = """\ -The @pytask.mark.parametrize decorator is deprecated and will be removed in pytask \ -v0.4. Either upgrade your code to the new syntax explained in \ -https://tinyurl.com/pytask-loops or silence the warning by setting \ -`silence_parametrize_deprecation = true` in your pyproject.toml under \ -[tool.pytask.ini_options] and pin pytask to <0.4. -""" - - @hookimpl def pytask_collect_file( session: Session, path: Path, reports: list[CollectionReport] @@ -129,26 +119,11 @@ def pytask_collect_file( if has_mark(obj, "task"): continue - if has_mark(obj, "parametrize"): - if not session.config.get("silence_parametrize_deprecation", False): - warnings.warn( - message=_PARAMETRIZE_DEPRECATION_WARNING, - category=FutureWarning, - stacklevel=1, - ) - - names_and_objects = session.hook.pytask_parametrize_task( - session=session, name=name, obj=obj - ) - else: - names_and_objects = [(name, obj)] - - for name_, obj_ in names_and_objects: - report = session.hook.pytask_collect_task_protocol( - session=session, reports=reports, path=path, name=name_, obj=obj_ - ) - if report is not None: - collected_reports.append(report) + report = session.hook.pytask_collect_task_protocol( + session=session, reports=reports, path=path, name=name, obj=obj + ) + if report is not None: + collected_reports.append(report) return collected_reports return None diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 9a49206f..011feb6e 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -8,7 +8,6 @@ import pathlib from typing import Any -from typing import Callable from typing import TYPE_CHECKING import click @@ -211,27 +210,6 @@ def pytask_collect_log( """ -# Hooks to parametrize tasks. - - -@hookspec(firstresult=True) -def pytask_parametrize_task( - session: Session, name: str, obj: Any -) -> list[tuple[str, Callable[..., Any]]]: - """Generate multiple tasks from name and object with parametrization.""" - - -@hookspec -def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[Any, Any]) -> None: - """Add some keyword arguments as markers to object. - - This hook moves arguments defined in the parametrization to marks of the same - function. This allows an argument like ``depends_on`` be transformed to the usual - ``@pytask.mark.depends_on`` marker which receives special treatment. - - """ - - # Hooks for resolving dependencies. diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index d4a77047..4dae51e6 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -196,11 +196,13 @@ def __getattr__(self, name: str) -> MarkDecorator | Any: if self.config is not None and name not in self.config["markers"]: if self.config["strict_markers"]: raise ValueError(f"Unknown pytask.mark.{name}.") - # Raise a specific error for common misspellings of "parametrize". - if name in ("parameterize", "parametrise", "parameterise"): - warnings.warn( - f"Unknown {name!r} mark, did you mean 'parametrize'?", stacklevel=1 - ) + + if name in ("parametrize", "parameterize", "parametrise", "parameterise"): + raise NotImplementedError( + "@pytask.mark.parametrize has been removed since pytask v0.4. " + "Upgrade your parametrized tasks to the new syntax defined in" + "https://tinyurl.com/pytask-loops or revert to v0.3." + ) from None warnings.warn( f"Unknown pytask.mark.{name} - is this a typo? You can register " diff --git a/src/_pytask/parametrize.py b/src/_pytask/parametrize.py deleted file mode 100644 index f3a9c2ca..00000000 --- a/src/_pytask/parametrize.py +++ /dev/null @@ -1,390 +0,0 @@ -"""This module contains the code for the parametrize plugin.""" -from __future__ import annotations - -import copy -import functools -import itertools -import pprint -import types -from typing import Any -from typing import Callable -from typing import Iterable -from typing import Sequence - -from _pytask.config import hookimpl -from _pytask.console import format_strings_as_flat_tree -from _pytask.console import TASK_ICON -from _pytask.mark import Mark -from _pytask.mark import MARK_GEN as mark # noqa: N811 -from _pytask.mark_utils import remove_marks -from _pytask.parametrize_utils import arg_value_to_id_component -from _pytask.session import Session -from _pytask.shared import find_duplicates - - -def parametrize( - arg_names: str | list[str] | tuple[str, ...], - arg_values: Iterable[Sequence[Any] | Any], - *, - ids: None | (Iterable[None | str | float | int | bool] | Callable[..., Any]) = None, -) -> tuple[ - str | list[str] | tuple[str, ...], - Iterable[Sequence[Any] | Any], - Iterable[None | str | float | int | bool] | Callable[..., Any] | None, -]: - """Parametrize a task function. - - Parametrizing a task allows to execute the same task with different arguments. - - Parameters - ---------- - arg_names : str | list[str] | tuple[str, ...] - The names of the arguments which can either be given as a comma-separated - string, a tuple of strings, or a list of strings. - arg_values : Iterable[Sequence[Any] | Any] - The values which correspond to names in ``arg_names``. For one argument, it is a - single iterable. For multiple argument names it is an iterable of iterables. - ids - This argument can either be a list with ids or a function which is called with - every value passed to the parametrized function. - - If you pass an iterable with ids, make sure to only use :obj:`bool`, - :obj:`float`, :obj:`int`, or :obj:`str` as values which are used to create task - ids like ``"task_dummpy.py::task_dummy[first_task_id]"``. - - If you pass a function, the function receives each value of the parametrization - and may return a boolean, number, string or None. For the latter, the - auto-generated value is used. - - """ - return arg_names, arg_values, ids - - -@hookimpl -def pytask_parse_config(config: dict[str, Any]) -> None: - """Add the marker to the config.""" - config["markers"]["parametrize"] = ( - "The marker for pytest's way of repeating tasks which is explained in this " - "tutorial: [link https://bit.ly/3uqZqkk]https://bit.ly/3uqZqkk[/]." - ) - - -@hookimpl -def pytask_parametrize_task( - session: Session, name: str, obj: Callable[..., Any] -) -> list[tuple[str, Callable[..., Any]]]: - """Parametrize a task. - - This function takes a single Python function and all parametrize decorators and - generates multiple instances of the same task with different arguments. - - Note that, while a single ``@pytask.mark.parametrize`` is handled like a loop or a - :func:`zip`, multiple ``@pytask.mark.parametrize`` decorators form a Cartesian - product. - - We cannot raise an error if the function does not use parametrized arguments since - some plugins will replace functions with their own implementation like pytask-r. - - """ - if callable(obj): - obj, markers = remove_marks(obj, "parametrize") # type: ignore[assignment] - - if len(markers) > 1: - raise NotImplementedError( - "You cannot apply @pytask.mark.parametrize multiple times to a task. " - "Use multiple for-loops, itertools.product or a different strategy to " - "create all combinations of inputs and pass it to a single " - "@pytask.mark.parametrize.\n\nFor improved readability, consider to " - "move the creation of inputs into its own function as shown in the " - "best-practices guide on parametrizations: https://pytask-dev.rtfd.io/" - "en/stable/how_to_guides/bp_scalable_repetitions_of_tasks.html." - ) - - base_arg_names, arg_names, arg_values = _parse_parametrize_markers( - markers, name - ) - - product_arg_names = list(itertools.product(*arg_names)) - product_arg_values = list(itertools.product(*arg_values)) - - names_and_functions: list[tuple[str, Callable[..., Any]]] = [] - for names, values in zip(product_arg_names, product_arg_values): - kwargs = dict( - zip( - itertools.chain.from_iterable(base_arg_names), - itertools.chain.from_iterable(values), - ) - ) - - # Copy function and attributes to allow in-place changes. - func = _copy_func(obj) # type: ignore[arg-type] - func.pytask_meta = copy.deepcopy( # type: ignore[attr-defined] - obj.pytask_meta # type: ignore[attr-defined] - ) - # Convert parametrized dependencies and products to decorator. - session.hook.pytask_parametrize_kwarg_to_marker(obj=func, kwargs=kwargs) - - func.pytask_meta.kwargs = { # type: ignore[attr-defined] - **func.pytask_meta.kwargs, # type: ignore[attr-defined] - **kwargs, - } - - name_ = f"{name}[{'-'.join(itertools.chain.from_iterable(names))}]" - names_and_functions.append((name_, func)) - - all_names = [i[0] for i in names_and_functions] - duplicates = find_duplicates(all_names) - - if duplicates: - text = format_strings_as_flat_tree( - duplicates, "Duplicated task ids", TASK_ICON - ) - raise ValueError( - "The following ids are duplicated while parametrizing task " - f"{name!r}.\n\n{text}\n\nIt might be caused by " - "parametrizing the task with the same combination of arguments " - "multiple times. Change the arguments or change the ids generated by " - "the parametrization." - ) - - return names_and_functions - return None - - -def _parse_parametrize_marker( - marker: Mark, name: str -) -> tuple[tuple[str, ...], list[tuple[str, ...]], list[tuple[Any, ...]]]: - """Parse parametrize marker. - - Parameters - ---------- - marker : Mark - A parametrize mark. - name : str - The name of the task function which is parametrized. - - Returns - ------- - base_arg_names : Tuple[str, ...] - Contains the names of the arguments. - processed_arg_names : List[Tuple[str, ...]] - Each tuple in the list represents the processed names of the arguments suffixed - with a number indicating the iteration. - processed_arg_values : List[Tuple[Any, ...]] - Each tuple in the list represents the values of the arguments for each - iteration. - - """ - arg_names, arg_values, ids = parametrize(*marker.args, **marker.kwargs) - - parsed_arg_names = _parse_arg_names(arg_names) - has_single_arg = len(parsed_arg_names) == 1 - parsed_arg_values = _parse_arg_values(arg_values, has_single_arg) - - _check_if_n_arg_names_matches_n_arg_values( - parsed_arg_names, parsed_arg_values, name - ) - - expanded_arg_names = _create_parametrize_ids_components( - parsed_arg_names, parsed_arg_values, ids - ) - - return parsed_arg_names, expanded_arg_names, parsed_arg_values - - -def _parse_parametrize_markers( - markers: list[Mark], name: str -) -> tuple[ - list[tuple[str, ...]], - list[list[tuple[str, ...]]], - list[list[tuple[Any, ...]]], -]: - """Parse parametrize markers.""" - parsed_markers = [_parse_parametrize_marker(marker, name) for marker in markers] - base_arg_names = [i[0] for i in parsed_markers] - processed_arg_names = [i[1] for i in parsed_markers] - processed_arg_values = [i[2] for i in parsed_markers] - - return base_arg_names, processed_arg_names, processed_arg_values - - -def _parse_arg_names(arg_names: str | list[str] | tuple[str, ...]) -> tuple[str, ...]: - """Parse arg_names argument of parametrize decorator. - - There are three allowed formats: - - 1. comma-separated string representation. - 2. a tuple of strings. - 3. a list of strings. - - All formats are converted to a tuple of strings. - - Parameters - ---------- - arg_names : Union[str, List[str], Tuple[str, ...]] - The names of the arguments which are parametrized. - - Returns - ------- - out : Tuple[str, ...] - The parsed arg_names. - - Example - ------- - >>> _parse_arg_names("i") - ('i',) - >>> _parse_arg_names("i, j") - ('i', 'j') - - """ - if isinstance(arg_names, str): - out = tuple(i.strip() for i in arg_names.split(",")) - elif isinstance(arg_names, (tuple, list)): - out = tuple(arg_names) - else: - raise TypeError( - "The argument 'arg_names' accepts comma-separated strings, tuples and lists" - f" of strings. It cannot accept {arg_names} with type {type(arg_names)}." - ) - - return out - - -def _parse_arg_values( - arg_values: Iterable[Sequence[Any] | Any], has_single_arg: bool -) -> list[tuple[Any, ...]]: - """Parse the values provided for each argument name. - - After processing the values, the return is a list where each value is an iteration - of the parametrization. Each iteration is a tuple of all parametrized arguments. - - Example - ------- - >>> _parse_arg_values(["a", "b", "c"], has_single_arg=True) - [('a',), ('b',), ('c',)] - >>> _parse_arg_values([(0, 0), (0, 1), (1, 0)], has_single_arg=False) - [(0, 0), (0, 1), (1, 0)] - - """ - return [ - tuple(i) - if isinstance(i, Iterable) - and not isinstance(i, str) - and not (isinstance(i, dict) and has_single_arg) - else (i,) - for i in arg_values - ] - - -def _check_if_n_arg_names_matches_n_arg_values( - arg_names: tuple[str, ...], arg_values: list[tuple[Any, ...]], name: str -) -> None: - """Check if the number of argument names matches the number of arguments.""" - n_names = len(arg_names) - n_values = [len(i) for i in arg_values] - unique_n_values = tuple(set(n_values)) - - if not all(i == n_names for i in unique_n_values): - pretty_arg_values = ( - f"{unique_n_values[0]}" - if len(unique_n_values) == 1 - else " or ".join(map(str, unique_n_values)) - ) - idx_example = [i == n_names for i in n_values].index(False) - formatted_example = pprint.pformat(arg_values[idx_example]) - raise ValueError( - f"Task {name!r} is parametrized with {n_names} 'arg_names', {arg_names}, " - f"but the number of provided 'arg_values' is {pretty_arg_values}. For " - f"example, here are the values of parametrization no. {idx_example}:" - f"\n\n{formatted_example}" - ) - - -def _create_parametrize_ids_components( - arg_names: tuple[str, ...], - arg_values: list[tuple[Any, ...]], - ids: None | (Iterable[None | str | float | int | bool] | Callable[..., Any]), -) -> list[tuple[str, ...]]: - """Create the ids for each parametrization. - - Parameters - ---------- - arg_names : Tuple[str, ...] - The names of the arguments of the parametrized function. - arg_values : List[Tuple[Any, ...]] - A list of tuples where each tuple is for one run. - ids - The ids associated with one parametrization. - - Examples - -------- - >>> _create_parametrize_ids_components(["i"], [(0,), (1,)], None) - [('0',), ('1',)] - - >>> _create_parametrize_ids_components(["i", "j"], [(0, (0,)), (1, (1,))], None) - [('0', 'j0'), ('1', 'j1')] - - """ - if isinstance(ids, Iterable): - raw_ids = [(id_,) for id_ in ids] - - if len(raw_ids) != len(arg_values): - raise ValueError("The number of ids must match the number of runs.") - - if not all( - isinstance(id_, (bool, int, float, str)) or id_ is None - for id_ in itertools.chain.from_iterable(raw_ids) - ): - raise ValueError( - "Ids for parametrization can only be of type bool, float, int, str or " - "None." - ) - - parsed_ids: list[tuple[str, ...]] = [ - (str(id_),) for id_ in itertools.chain.from_iterable(raw_ids) - ] - - else: - parsed_ids = [] - for i, _arg_values in enumerate(arg_values): - id_components = tuple( - arg_value_to_id_component(arg_names[j], arg_value, i, ids) - for j, arg_value in enumerate(_arg_values) - ) - parsed_ids.append(id_components) - - return parsed_ids - - -@hookimpl -def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[str, str]) -> None: - """Add some parametrized keyword arguments as decorator.""" - if callable(obj): - for marker_name in ("depends_on", "produces"): - if marker_name in kwargs: - mark.__getattr__(marker_name)(kwargs.pop(marker_name))(obj) - - -def _copy_func(func: types.FunctionType) -> types.FunctionType: - """Create a copy of a function. - - Based on https://stackoverflow.com/a/13503277/7523785. - - Example - ------- - >>> def _func(): pass - >>> copied_func = _copy_func(_func) - >>> _func is copied_func - False - - """ - new_func = types.FunctionType( - func.__code__, - func.__globals__, - name=func.__name__, - argdefs=func.__defaults__, - closure=func.__closure__, - ) - new_func = functools.update_wrapper(new_func, func) - new_func.__kwdefaults__ = func.__kwdefaults__ - return new_func diff --git a/src/_pytask/parametrize_utils.py b/src/_pytask/parametrize_utils.py deleted file mode 100644 index f259c7b3..00000000 --- a/src/_pytask/parametrize_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from typing import Any -from typing import Callable - - -def arg_value_to_id_component( - arg_name: str, arg_value: Any, i: int, id_func: Callable[..., Any] | None -) -> str: - """Create id component from the name and value of the argument. - - First, transform the value of the argument with a user-defined function if given. - Otherwise, take the original value. Then, if the value is a :obj:`bool`, - :obj:`float`, :obj:`int`, or :obj:`str`, cast it to a string. Otherwise, define a - placeholder value from the name of the argument and the iteration. - - Parameters - ---------- - arg_name : str - Name of the parametrized function argument. - arg_value : Any - Value of the argument. - i : int - The ith iteration of the parametrization. - id_func : Union[Callable[..., Any], None] - A callable which maps argument values to :obj:`bool`, :obj:`float`, :obj:`int`, - or :obj:`str` or anything else. Any object with a different dtype than the first - will be mapped to an auto-generated id component. - - Returns - ------- - id_component : str - A part of the final parametrized id. - - """ - id_component = id_func(arg_value) if id_func is not None else None - if isinstance(id_component, (bool, float, int, str)): - id_component = str(id_component) - elif isinstance(arg_value, (bool, float, int, str)): - id_component = str(arg_value) - else: - id_component = arg_name + str(i) - return id_component diff --git a/src/_pytask/task.py b/src/_pytask/task.py index d71cb668..7b806af7 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -5,7 +5,6 @@ from typing import Any from _pytask.config import hookimpl -from _pytask.mark_utils import has_mark from _pytask.report import CollectionReport from _pytask.session import Session from _pytask.task_utils import COLLECTED_TASKS @@ -39,24 +38,11 @@ def pytask_collect_file( collected_reports = [] for name, function in name_to_function.items(): - session.hook.pytask_parametrize_kwarg_to_marker( - obj=function, - kwargs=function.pytask_meta.kwargs, # type: ignore[attr-defined] + report = session.hook.pytask_collect_task_protocol( + session=session, reports=reports, path=path, name=name, obj=function ) - - if has_mark(function, "parametrize"): - names_and_objects = session.hook.pytask_parametrize_task( - session=session, name=name, obj=function - ) - else: - names_and_objects = [(name, function)] - - for name_, obj_ in names_and_objects: - report = session.hook.pytask_collect_task_protocol( - session=session, reports=reports, path=path, name=name_, obj=obj_ - ) - if report is not None: - collected_reports.append(report) + if report is not None: + collected_reports.append(report) return collected_reports return None diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index e8e6bd4e..ad541c9d 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -9,7 +9,6 @@ from _pytask.mark import Mark from _pytask.models import CollectionMetadata -from _pytask.parametrize_utils import arg_value_to_id_component from _pytask.shared import find_duplicates @@ -51,6 +50,13 @@ def task( """ def wrapper(func: Callable[..., Any]) -> None: + for arg, arg_name in ((name, "name"), (id, "id")): + if not (isinstance(arg, str) or arg is None): + raise ValueError( + f"Argument {arg_name!r} of @pytask.mark.task must be a str, but it " + f"is {arg!r}." + ) + unwrapped = inspect.unwrap(func) raw_path = inspect.getfile(unwrapped) @@ -104,6 +110,15 @@ def parse_collected_tasks_with_task_marker( else: collected_tasks[name] = [i[1] for i in parsed_tasks if i[0] == name][0] + # TODO: Remove when parsing dependencies and products from all arguments is + # implemented. + for task in collected_tasks.values(): + meta = task.pytask_meta # type: ignore[attr-defined] + for marker_name in ("depends_on", "produces"): + if marker_name in meta.kwargs: + value = meta.kwargs.pop(marker_name) + meta.markers.append(Mark(marker_name, (value,), {})) + return collected_tasks @@ -169,7 +184,7 @@ def _generate_ids_for_tasks( id_ = f"{name}[{i}]" else: stringified_args = [ - arg_value_to_id_component( + _arg_value_to_id_component( arg_name=parameter, arg_value=task.pytask_meta.kwargs.get( # type: ignore[attr-defined] parameter @@ -183,3 +198,42 @@ def _generate_ids_for_tasks( id_ = f"{name}[{id_}]" out[id_] = task return out + + +def _arg_value_to_id_component( + arg_name: str, arg_value: Any, i: int, id_func: Callable[..., Any] | None +) -> str: + """Create id component from the name and value of the argument. + + First, transform the value of the argument with a user-defined function if given. + Otherwise, take the original value. Then, if the value is a :obj:`bool`, + :obj:`float`, :obj:`int`, or :obj:`str`, cast it to a string. Otherwise, define a + placeholder value from the name of the argument and the iteration. + + Parameters + ---------- + arg_name : str + Name of the parametrized function argument. + arg_value : Any + Value of the argument. + i : int + The ith iteration of the parametrization. + id_func : Union[Callable[..., Any], None] + A callable which maps argument values to :obj:`bool`, :obj:`float`, :obj:`int`, + or :obj:`str` or anything else. Any object with a different dtype than the first + will be mapped to an auto-generated id component. + + Returns + ------- + id_component : str + A part of the final parametrized id. + + """ + id_component = id_func(arg_value) if id_func is not None else None + if isinstance(id_component, (bool, float, int, str)): + id_component = str(id_component) + elif isinstance(arg_value, (bool, float, int, str)): + id_component = str(arg_value) + else: + id_component = arg_name + str(i) + return id_component diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index 609594f9..eba87f4c 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -81,10 +81,12 @@ def test_collect_parametrized_tasks(runner, tmp_path): source = """ import pytask - @pytask.mark.depends_on("in.txt") - @pytask.mark.parametrize("arg, produces", [(0, "out_0.txt"), (1, "out_1.txt")]) - def task_example(arg): - pass + for arg, produces in [(0, "out_0.txt"), (1, "out_1.txt")]: + + @pytask.mark.task + @pytask.mark.depends_on("in.txt") + def task_example(arg=arg, produces=produces): + pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").touch() diff --git a/tests/test_live.py b/tests/test_live.py index 851a11a4..abcfbacf 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -262,9 +262,11 @@ def test_full_execution_table_is_displayed_at_the_end_of_execution(tmp_path, run source = """ import pytask - @pytask.mark.parametrize("produces", [f"{i}.txt" for i in range(4)]) - def task_create_file(produces): - produces.touch() + for produces in [f"{i}.txt" for i in range(4)]: + + @pytask.mark.task + def task_create_file(produces=produces): + produces.touch() """ # Subfolder to reduce task id and be able to check the output later. tmp_path.joinpath("d").mkdir() diff --git a/tests/test_mark.py b/tests/test_mark.py index 2b79367b..32d468b1 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -172,16 +172,16 @@ def task_no_2(): ], ) def test_keyword_option_parametrize(tmp_path, expr: str, expected_passed: str) -> None: - tmp_path.joinpath("task_module.py").write_text( - textwrap.dedent( - """ - import pytask - @pytask.mark.parametrize("arg", [None, 1.3, "2-3"]) - def task_func(arg): - pass - """ - ) - ) + source = """ + import pytask + + for arg in [None, 1.3, "2-3"]: + + @pytask.mark.task + def task_func(arg=arg): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) session = main({"paths": tmp_path, "expression": expr}) assert session.exit_code == ExitCode.OK diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py deleted file mode 100644 index 37994a8b..00000000 --- a/tests/test_parametrize.py +++ /dev/null @@ -1,531 +0,0 @@ -from __future__ import annotations - -import itertools -import textwrap -from contextlib import ExitStack as does_not_raise # noqa: N813 -from typing import NamedTuple - -import _pytask.parametrize -import pytask -import pytest -from _pytask.parametrize import _check_if_n_arg_names_matches_n_arg_values -from _pytask.parametrize import _parse_arg_names -from _pytask.parametrize import _parse_arg_values -from _pytask.parametrize import _parse_parametrize_markers -from _pytask.parametrize import pytask_parametrize_task -from _pytask.pluginmanager import get_plugin_manager -from pytask import cli -from pytask import ExitCode -from pytask import main -from pytask import Mark -from pytask import Session - - -@pytest.fixture(scope="module") -def session(): - pm = get_plugin_manager() - pm.register(_pytask.parametrize) - session = Session(hook=pm.hook) - return session - - -@pytest.mark.integration() -def test_pytask_generate_tasks_0(session): - @pytask.mark.parametrize("i", range(2)) - def func(i): # noqa: ARG001, pragma: no cover - pass - - names_and_objs = pytask_parametrize_task(session, "func", func) - - assert [i[0] for i in names_and_objs] == ["func[0]", "func[1]"] - assert names_and_objs[0][1].pytask_meta.kwargs["i"] == 0 - assert names_and_objs[1][1].pytask_meta.kwargs["i"] == 1 - - -@pytest.mark.integration() -@pytest.mark.xfail(strict=True, reason="Cartesian task product is disabled.") -def test_pytask_generate_tasks_1(session): - @pytask.mark.parametrize("j", range(2)) - @pytask.mark.parametrize("i", range(2)) - def func(i, j): # noqa: ARG001, pragma: no cover - pass - - pytask_parametrize_task(session, "func", func) - - -@pytest.mark.integration() -@pytest.mark.xfail(strict=True, reason="Cartesian task product is disabled.") -def test_pytask_generate_tasks_2(session): - @pytask.mark.parametrize("j, k", itertools.product(range(2), range(2))) - @pytask.mark.parametrize("i", range(2)) - def func(i, j, k): # noqa: ARG001, pragma: no cover - pass - - pytask_parametrize_task(session, "func", func) - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("arg_names", "expected"), - [ - ("i", ("i",)), - ("i,j", ("i", "j")), - ("i, j", ("i", "j")), - (("i", "j"), ("i", "j")), - (["i", "j"], ("i", "j")), - ], -) -def test_parse_arg_names(arg_names, expected): - parsed_arg_names = _parse_arg_names(arg_names) - assert parsed_arg_names == expected - - -class TaskArguments(NamedTuple): - a: int - b: int - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("arg_values", "has_single_arg", "expected"), - [ - (["a", "b", "c"], True, [("a",), ("b",), ("c",)]), - ([(0, 0), (0, 1), (1, 0)], False, [(0, 0), (0, 1), (1, 0)]), - ([[0, 0], [0, 1], [1, 0]], False, [(0, 0), (0, 1), (1, 0)]), - ({"a": 0, "b": 1}, False, [("a",), ("b",)]), - ([{"a": 0, "b": 1}], True, [({"a": 0, "b": 1},)]), - ([TaskArguments(1, 2)], False, [(1, 2)]), - ([TaskArguments(a=1, b=2)], False, [(1, 2)]), - ([TaskArguments(b=2, a=1)], False, [(1, 2)]), - ], -) -def test_parse_arg_values(arg_values, has_single_arg, expected): - parsed_arg_values = _parse_arg_values(arg_values, has_single_arg) - assert parsed_arg_values == expected - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("arg_names", "expectation"), - [ - ("i", does_not_raise()), - ("i, j", does_not_raise()), - (("i", "j"), does_not_raise()), - (["i", "j"], does_not_raise()), - (range(1, 2), pytest.raises(TypeError)), - ({"i": None, "j": None}, pytest.raises(TypeError)), - ({"i", "j"}, pytest.raises(TypeError)), - ], -) -def test_parse_argnames_raise_error(arg_names, expectation): - with expectation: - _parse_arg_names(arg_names) - - -@pytest.mark.integration() -@pytest.mark.parametrize( - ("markers", "exp_base_arg_names", "exp_arg_names", "exp_arg_values"), - [ - ( - [ - Mark("parametrize", ("i", range(2)), {}), - Mark("parametrize", ("j", range(2)), {}), - ], - [("i",), ("j",)], - [[("0",), ("1",)], [("0",), ("1",)]], - [[(0,), (1,)], [(0,), (1,)]], - ), - ( - [Mark("parametrize", ("i", range(3)), {})], - [("i",)], - [[("0",), ("1",), ("2",)]], - [[(0,), (1,), (2,)]], - ), - ], -) -def test_parse_parametrize_markers( - markers, exp_base_arg_names, exp_arg_names, exp_arg_values -): - base_arg_names, arg_names, arg_values = _parse_parametrize_markers(markers, "task_") - - assert base_arg_names == exp_base_arg_names - assert arg_names == exp_arg_names - assert arg_values == exp_arg_values - - -@pytest.mark.end_to_end() -def test_parametrizing_tasks(tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i, produces', [(1, "1.txt"), (2, "2.txt")]) - def task_write_numbers_to_file(produces, i): - produces.write_text(str(i)) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.OK - for i in range(1, 3): - assert tmp_path.joinpath(f"{i}.txt").read_text() == str(i) - - -@pytest.mark.end_to_end() -def test_parametrizing_dependencies_and_targets(tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i, produces', [(1, "1.txt"), (2, "2.txt")]) - def task_save_numbers(i, produces): - produces.write_text(str(i)) - - @pytask.mark.parametrize("depends_on, produces", [ - ("1.txt", "1_out.txt"), ("2.txt", "2_out.txt") - ]) - def task_save_numbers_again(depends_on, produces): - produces.write_text(depends_on.read_text()) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.OK - - -@pytest.mark.end_to_end() -def test_parametrize_iterator(tmp_path): - """`parametrize` should work with generators.""" - source = """ - import pytask - def gen(): - yield 1 - yield 2 - yield 3 - @pytask.mark.parametrize('a', gen()) - def task_func(a): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) - assert session.exit_code == ExitCode.OK - assert len(session.execution_reports) == 3 - - -@pytest.mark.end_to_end() -def test_raise_error_if_function_does_not_use_parametrized_arguments(tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i', range(2)) - def task_func(): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.FAILED - assert isinstance(session.execution_reports[0].exc_info[1], TypeError) - assert isinstance(session.execution_reports[1].exc_info[1], TypeError) - - -@pytest.mark.end_to_end() -@pytest.mark.parametrize( - ("arg_values", "ids"), - [ - (range(2), ["first_trial", "second_trial"]), - ([True, False], ["first_trial", "second_trial"]), - ], -) -def test_parametrize_w_ids(tmp_path, arg_values, ids): - tmp_path.joinpath("task_module.py").write_text( - textwrap.dedent( - f""" - import pytask - - @pytask.mark.parametrize('i', {arg_values}, ids={ids}) - def task_func(i): - pass - """ - ) - ) - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.OK - for task, id_ in zip(session.tasks, ids): - assert id_ in task.name - - -@pytest.mark.end_to_end() -def test_two_parametrize_w_ids(runner, tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i', range(2), ids=["2.1", "2.2"]) - @pytask.mark.parametrize('j', range(2), ids=["1.1", "1.2"]) - def task_func(i, j): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.COLLECTION_FAILED - assert "You cannot apply @pytask.mark.parametrize multiple" in result.output - - -@pytest.mark.end_to_end() -@pytest.mark.parametrize("ids", [["a"], list("abc"), ((1,), (2,)), ({0}, {1})]) -def test_raise_error_for_irregular_ids(tmp_path, ids): - tmp_path.joinpath("task_module.py").write_text( - textwrap.dedent( - f""" - import pytask - - @pytask.mark.parametrize('i', range(2), ids={ids}) - def task_func(): - pass - """ - ) - ) - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert isinstance(session.collection_reports[0].exc_info[1], ValueError) - - -@pytest.mark.end_to_end() -def test_raise_error_if_parametrization_produces_non_unique_tasks(tmp_path): - tmp_path.joinpath("task_module.py").write_text( - textwrap.dedent( - """ - import pytask - - @pytask.mark.parametrize('i', [0, 0]) - def task_func(i): - pass - """ - ) - ) - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert isinstance(session.collection_reports[0].exc_info[1], ValueError) - - -@pytest.mark.end_to_end() -@pytest.mark.parametrize( - ("arg_names", "arg_values", "content"), - [ - ( - ("i", "j"), - [1, 2, 3], - [ - "ValueError", - "with 2 'arg_names', ('i', 'j'),", - "'arg_values' is 1.", - "parametrization no. 0:", - "(1,)", - ], - ), - ( - ("i", "j"), - [(1, 2, 3)], - [ - "ValueError", - "with 2 'arg_names', ('i', 'j'),", - "'arg_values' is 3.", - "parametrization no. 0:", - "(1, 2, 3)", - ], - ), - ( - ("i", "j"), - [(1, 2), (1, 2, 3)], - [ - "ValueError", - "with 2 'arg_names', ('i', 'j'),", - "'arg_values' is 2 or 3.", - "parametrization no. 1:", - "(1, 2, 3)", - ], - ), - ], -) -def test_wrong_number_of_names_and_wrong_number_of_arguments( - tmp_path, runner, arg_names, arg_values, content -): - source = f""" - import pytask - - @pytask.mark.parametrize({arg_names}, {arg_values}) - def task_func(): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.COLLECTION_FAILED - for c in content: - assert c in result.output - - -@pytest.mark.end_to_end() -def test_generators_are_removed_from_depends_on_produces(tmp_path): - source = """ - from pathlib import Path - import pytask - - @pytask.mark.parametrize("produces", [ - ((x for x in ["out.txt", "out_2.txt"]),), - ["in.txt"], - ]) - def task_example(produces): - produces = {0: produces} if isinstance(produces, Path) else produces - for p in produces.values(): - p.write_text("hihi") - """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - - session = main({"paths": tmp_path}) - assert session.exit_code == ExitCode.OK - assert session.tasks[0].function.pytask_meta.markers == [] - - -@pytest.mark.end_to_end() -def test_parametrizing_tasks_with_namedtuples(runner, tmp_path): - source = """ - from typing import NamedTuple - import pytask - from pathlib import Path - - - class Task(NamedTuple): - i: int - produces: Path - - - @pytask.mark.parametrize('i, produces', [ - Task(i=1, produces="1.txt"), Task(produces="2.txt", i=2), - ]) - def task_write_numbers_to_file(produces, i): - produces.write_text(str(i)) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.OK - for i in range(1, 3): - assert tmp_path.joinpath(f"{i}.txt").read_text() == str(i) - - -@pytest.mark.end_to_end() -def test_parametrization_with_different_n_of_arg_names_and_arg_values(runner, tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i, produces', [(1, "1.txt"), (2, 3, "2.txt")]) - def task_write_numbers_to_file(produces, i): - produces.write_text(str(i)) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.COLLECTION_FAILED - assert "Task 'task_write_numbers_to_file' is parametrized with 2" in result.output - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("arg_names", "arg_values", "name", "expectation"), - [ - pytest.param( - ("a",), - [(1,), (2,)], - "task_name", - does_not_raise(), - id="normal one argument parametrization", - ), - pytest.param( - ("a", "b"), - [(1, 2), (3, 4)], - "task_name", - does_not_raise(), - id="normal two argument argument parametrization", - ), - pytest.param( - ("a",), - [(1, 2), (2,)], - "task_name", - pytest.raises(ValueError, match="Task 'task_name' is parametrized with 1"), - id="error with one argument parametrization", - ), - pytest.param( - ("a", "b"), - [(1, 2), (3, 4, 5)], - "task_name", - pytest.raises(ValueError, match="Task 'task_name' is parametrized with 2"), - id="error with two argument argument parametrization", - ), - ], -) -def test_check_if_n_arg_names_matches_n_arg_values( - arg_names, arg_values, name, expectation -): - with expectation: - _check_if_n_arg_names_matches_n_arg_values(arg_names, arg_values, name) - - -@pytest.mark.end_to_end() -def test_parametrize_with_single_dict(tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i', [{"a": 1}, {"a": 1.0}]) - def task_write_numbers_to_file(i): - assert i["a"] == 1 - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.OK - - -@pytest.mark.end_to_end() -def test_deprecation_warning_for_parametrizing_tasks(runner, tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i, produces', [(1, "1.txt"), (2, "2.txt")]) - def task_write_numbers_to_file(produces, i): - produces.write_text(str(i)) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.OK - assert "FutureWarning" in result.output - - -@pytest.mark.end_to_end() -def test_silence_deprecation_warning_for_parametrizing_tasks(runner, tmp_path): - source = """ - import pytask - - @pytask.mark.parametrize('i, produces', [(1, "1.txt"), (2, "2.txt")]) - def task_write_numbers_to_file(produces, i): - produces.write_text(str(i)) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("pyproject.toml").write_text( - "[tool.pytask.ini_options]\nsilence_parametrize_deprecation = true" - ) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.OK - assert "FutureWarning" not in result.output diff --git a/tests/test_task.py b/tests/test_task.py index 33905c26..3313fdb7 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -33,35 +33,6 @@ def {func_name}(produces): assert session.tasks[0].name.endswith(f"task_module.py::{func_name}") -@pytest.mark.end_to_end() -@pytest.mark.parametrize("func_name", ["task_example", "func"]) -@pytest.mark.parametrize("task_name", ["the_only_task", None]) -def test_task_with_task_decorator_with_parametrize(tmp_path, func_name, task_name): - task_decorator_input = f"{task_name!r}" if task_name else task_name - source = f""" - import pytask - - @pytask.mark.task({task_decorator_input}) - @pytask.mark.parametrize("produces", ["out_1.txt", "out_2.txt"]) - def {func_name}(produces): - produces.write_text("Hello. It's me.") - """ - path_to_module = tmp_path.joinpath("task_module.py") - path_to_module.write_text(textwrap.dedent(source)) - - session = main({"paths": tmp_path}) - - assert session.exit_code == ExitCode.OK - - file_name = path_to_module.name - if task_name: - assert session.tasks[0].name.endswith(f"{file_name}::{task_name}[out_1.txt]") - assert session.tasks[1].name.endswith(f"{file_name}::{task_name}[out_2.txt]") - else: - assert session.tasks[0].name.endswith(f"{file_name}::{func_name}[out_1.txt]") - assert session.tasks[1].name.endswith(f"{file_name}::{func_name}[out_2.txt]") - - @pytest.mark.end_to_end() def test_parametrization_in_for_loop(tmp_path, runner): source = """ @@ -178,7 +149,7 @@ def test_parametrization_in_for_loop_with_ids(tmp_path, runner): for i in range(2): @pytask.mark.task( - "deco_task", id=i, kwargs={"i": i, "produces": f"out_{i}.txt"} + "deco_task", id=str(i), kwargs={"i": i, "produces": f"out_{i}.txt"} ) def example(produces, i): produces.write_text(str(i)) @@ -429,3 +400,41 @@ def task_example(): assert "task_example[0]" in result.output assert "task_example[1]" in result.output assert "Collected 2 tasks" in result.output + + +@pytest.mark.end_to_end() +@pytest.mark.parametrize( + "irregular_id", [1, (1,), [1], {1}, ["a"], list("abc"), ((1,), (2,)), ({0}, {1})] +) +def test_raise_errors_for_irregular_ids(runner, tmp_path, irregular_id): + source = f""" + import pytask + + @pytask.mark.task(id={irregular_id}) + def task_example(): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "Argument 'id' of @pytask.mark.task" in result.output + + +@pytest.mark.end_to_end() +@pytest.mark.xfail(reason="Should fail. Mandatory products will fix the issue.") +def test_raise_error_if_parametrization_produces_non_unique_tasks(tmp_path): + source = """ + import pytask + + for i in [0, 0]: + @pytask.mark.task(id=str(i)) + def task_func(i=i): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.COLLECTION_FAILED + assert isinstance(session.collection_reports[0].exc_info[1], ValueError) diff --git a/tests/test_parametrize_utils.py b/tests/test_task_utils.py similarity index 84% rename from tests/test_parametrize_utils.py rename to tests/test_task_utils.py index 8b483208..779cccd1 100644 --- a/tests/test_parametrize_utils.py +++ b/tests/test_task_utils.py @@ -1,7 +1,7 @@ from __future__ import annotations import pytest -from _pytask.parametrize_utils import arg_value_to_id_component +from _pytask.task_utils import _arg_value_to_id_component @pytest.mark.unit() @@ -22,5 +22,5 @@ ], ) def test_arg_value_to_id_component(arg_name, arg_value, i, id_func, expected): - result = arg_value_to_id_component(arg_name, arg_value, i, id_func) + result = _arg_value_to_id_component(arg_name, arg_value, i, id_func) assert result == expected