diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c17e0e52..b5b51282c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,10 +33,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Run pre-commit - if: ${{ matrix.python-version == '3.10' }} - uses: pre-commit/action@v3.0.0 - - name: Cache poetry virtualenv uses: actions/cache@v3 with: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 000000000..8fc880057 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,35 @@ +name: Precommit and Mypy (soft) + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + + package: + name: "Precommit and Mypy (soft)" + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: pip + + - name: Run pre-commit (fail upon errors) + uses: pre-commit/action@v3.0.0 + + - name: Install mypy + run: pip install mypy + + - name: Run mypy (do not fail for errors) + continue-on-error: true + run: mypy --package fractal_tasks_core --ignore-missing-imports --warn-redundant-casts --warn-unused-ignores --warn-unreachable --pretty diff --git a/CHANGELOG.md b/CHANGELOG.md index 133519356..e4ff79d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,11 @@ * Create new `dev` subpackage (\#384). * Make tasks-related dependencies optional, and installable via `fractal-tasks` extra (\#390). * Remove `tools` package extra (\#384), and split the subpackage content into `lib_ROI_overlaps` and `examples` (\#390). -* Define models for complex task arguments: - * Introduce `lib_channels.Channel` (\#410). +* Introduce Pydantic models for task arguments (\#410, \#422): + * `lib_channels.OmeroChannel` (\#410, \#422); + * `tasks._input_models.Channel` (\#422); + * `tasks._input_models.NapariWorkflowsInput` (\#422); + * `tasks._input_models.NapariWorkflowsOutput` (\#422). * JSON Schemas for task arguments: * Add JSON schemas for task arguments in the package manifest (\#369, \#384). * Remove `TaskArguments` models and switch to Pydantic V1 `validate_arguments` (\#369). diff --git a/docs/source/development.rst b/docs/source/development.rst index d3afd65fa..efc1fd41f 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -21,6 +21,26 @@ We use `pytest `_ for unit and integration testing of F The tests files are in the ``tests`` folder of the repository, and they are also run on GitHub (with both python 3.8 and 3.9 versions). +Mypy +^^^^ +You can run ``mypy`` for instance as:: + + poetry run mypy --package fractal_tasks_core --ignore-missing-imports --warn-redundant-casts --warn-unused-ignores --warn-unreachable --pretty + + +Documentation +~~~~~~~~~~~~~ + +To build the documentation, you should first install the package with ``poetry +install --with docs``; then use one of:: + + # Static build at docs/build/index.html + poetry run sphinx-build docs/source docs/build -W + + # Automatically-updated build, at http://127.0.0.1:8000: + poetry run sphinx-autobuild docs/source docs/build -W + + How to release ~~~~~~~~~~~~~~ diff --git a/fractal_tasks_core/__FRACTAL_MANIFEST__.json b/fractal_tasks_core/__FRACTAL_MANIFEST__.json index 88cd43bb5..48f8ef3c8 100644 --- a/fractal_tasks_core/__FRACTAL_MANIFEST__.json +++ b/fractal_tasks_core/__FRACTAL_MANIFEST__.json @@ -45,9 +45,9 @@ "title": "Allowed Channels", "type": "array", "items": { - "$ref": "#/definitions/Channel" + "$ref": "#/definitions/OmeroChannel" }, - "description": "A list of channel dictionaries, where each channel must include the ``wavelength_id`` key and where the corresponding values should be unique across channels. # TODO: improve after Channel input refactor See issue 386" + "description": "A list of ``OmeroChannel``s, where each channel must include the ``wavelength_id`` attribute and where the ``wavelength_id`` values must be unique across the list." }, "num_levels": { "title": "Num Levels", @@ -76,8 +76,8 @@ ], "additionalProperties": false, "definitions": { - "ChannelWindow": { - "title": "ChannelWindow", + "Window": { + "title": "Window", "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n 1. ``min`` and ``max`` are optional, since we have custom logic to set\n their values.", "type": "object", "properties": { @@ -103,8 +103,8 @@ "end" ] }, - "Channel": { - "title": "Channel", + "OmeroChannel": { + "title": "OmeroChannel", "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md)\n\n 1. Additional attributes ``wavelength_id`` and ``index``.\n 2. We make ``color`` an optional attribute, since we have custom\n logic to set its value.\n 3. We make ``window`` an optional attribute, so that we can also\n process zarr arrays which do not have this attribute.", "type": "object", "properties": { @@ -117,7 +117,7 @@ "type": "integer" }, "window": { - "$ref": "#/definitions/ChannelWindow" + "$ref": "#/definitions/Window" }, "color": { "title": "Color", @@ -351,25 +351,13 @@ "type": "integer", "description": "Pyramid level of the image to be segmented. Choose 0 to process at full resolution." }, - "wavelength_id": { - "title": "Wavelength Id", - "type": "string", - "description": "Identifier of a channel based on the wavelength (e.g. ``A01_C01``). If not ``None``, then ``channel_label` must be ``None``." - }, - "channel_label": { - "title": "Channel Label", - "type": "string", - "description": "Identifier of a channel based on its label (e.g. ``DAPI``). If not ``None``, then ``wavelength_id`` must be ``None``." - }, - "wavelength_id_c2": { - "title": "Wavelength Id C2", - "type": "string", - "description": "Identifier of a second channel in the same format as the first wavelength_id. If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker." + "channel": { + "$ref": "#/definitions/Channel", + "description": "Primary channel for segmentation; requires either ``wavelength_id`` (e.g. ``A01_C01``) or ``label`` (e.g. ``DAPI``)." }, - "channel_label_c2": { - "title": "Channel Label C2", - "type": "string", - "description": "Identifier of a second channel in the same format as the first wavelength_id. If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker." + "channel2": { + "$ref": "#/definitions/Channel", + "description": "Second channel for segmentation (in the same format as ``channel``). If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker." }, "input_ROI_table": { "title": "Input Roi Table", @@ -463,9 +451,27 @@ "output_path", "component", "metadata", - "level" + "level", + "channel" ], - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "Channel": { + "title": "Channel", + "description": "A channel which is specified by either ``wavelength_id`` or ``label``.", + "type": "object", + "properties": { + "wavelength_id": { + "title": "Wavelength Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + } + } + } + } }, "executable": "tasks/cellpose_segmentation.py", "input_type": "zarr", @@ -585,23 +591,17 @@ "title": "Input Specs", "type": "object", "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "$ref": "#/definitions/NapariWorkflowsInput" }, - "description": "See examples above. TODO: Update after issue 404" + "description": "A dictionary of ``NapariWorkflowsInput`` values." }, "output_specs": { "title": "Output Specs", "type": "object", "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "$ref": "#/definitions/NapariWorkflowsOutput" }, - "description": "See examples above. TODO: Update after issue 404" + "description": "A dictionary of ``NapariWorkflowsOutput`` values." }, "input_ROI_table": { "title": "Input Roi Table", @@ -637,7 +637,75 @@ "input_specs", "output_specs" ], - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "Channel": { + "title": "Channel", + "description": "A channel which is specified by either ``wavelength_id`` or ``label``.", + "type": "object", + "properties": { + "wavelength_id": { + "title": "Wavelength Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + } + } + }, + "NapariWorkflowsInput": { + "title": "NapariWorkflowsInput", + "description": "A value of the ``input_specs`` argument in ``napari_workflows_wrapper``.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "enum": [ + "image", + "label" + ], + "type": "string" + }, + "label_name": { + "title": "Label Name", + "type": "string" + }, + "channel": { + "$ref": "#/definitions/Channel" + } + }, + "required": [ + "type" + ] + }, + "NapariWorkflowsOutput": { + "title": "NapariWorkflowsOutput", + "description": "A value of the ``output_specs`` argument in ``napari_workflows_wrapper``.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "enum": [ + "label", + "dataframe" + ], + "type": "string" + }, + "label_name": { + "title": "Label Name", + "type": "string" + }, + "table_name": { + "title": "Table Name", + "type": "string" + } + }, + "required": [ + "type" + ] + } + } }, "executable": "tasks/napari_workflows_wrapper.py", "input_type": "zarr", @@ -692,10 +760,10 @@ "additionalProperties": { "type": "array", "items": { - "$ref": "#/definitions/Channel" + "$ref": "#/definitions/OmeroChannel" } }, - "description": "A dictionary of channel dictionaries, where each channel must include the ``wavelength_id`` key and where the corresponding values should be unique across channels. Values are the integers of the channel order, i.e. ``\"0\"``, ``\"1\"`` etc. # TODO: improve after Channel input refactor https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/386" + "description": "A dictionary of lists of ``OmeroChannel``s, where each channel must include the ``wavelength_id`` attribute and where the ``wavelength_id`` values must be unique across each list. Dictionary keys represent channel indices (``\"0\",\"1\",..``)." }, "num_levels": { "title": "Num Levels", @@ -737,8 +805,8 @@ ], "additionalProperties": false, "definitions": { - "ChannelWindow": { - "title": "ChannelWindow", + "Window": { + "title": "Window", "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n 1. ``min`` and ``max`` are optional, since we have custom logic to set\n their values.", "type": "object", "properties": { @@ -764,8 +832,8 @@ "end" ] }, - "Channel": { - "title": "Channel", + "OmeroChannel": { + "title": "OmeroChannel", "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md)\n\n 1. Additional attributes ``wavelength_id`` and ``index``.\n 2. We make ``color`` an optional attribute, since we have custom\n logic to set its value.\n 3. We make ``window`` an optional attribute, so that we can also\n process zarr arrays which do not have this attribute.", "type": "object", "properties": { @@ -778,7 +846,7 @@ "type": "integer" }, "window": { - "$ref": "#/definitions/ChannelWindow" + "$ref": "#/definitions/Window" }, "color": { "title": "Color", diff --git a/fractal_tasks_core/lib_ROI_overlaps.py b/fractal_tasks_core/lib_ROI_overlaps.py index 4b146c52f..606547967 100644 --- a/fractal_tasks_core/lib_ROI_overlaps.py +++ b/fractal_tasks_core/lib_ROI_overlaps.py @@ -18,6 +18,7 @@ from typing import Callable from typing import Optional from typing import Sequence +from typing import Union import pandas as pd @@ -92,7 +93,9 @@ def is_overlapping_3D(box1, box2, tol=0) -> bool: return overlap_x and overlap_y and overlap_z -def get_overlapping_pair(tmp_df: pd.DataFrame, tol: float = 0) -> tuple[int]: +def get_overlapping_pair( + tmp_df: pd.DataFrame, tol: float = 0 +) -> Union[tuple[int, int], bool]: """ Finds the indices for the next overlapping FOVs pair @@ -363,7 +366,7 @@ def _is_overlapping_3D_int(box1: list[int], box2: list[int]) -> bool: def find_overlaps_in_ROI_indices( list_indices: list[list[int]], -) -> Optional[tuple[int]]: +) -> Optional[tuple[int, int]]: """ Given a list of integer ROI indices, find whether there are overlaps diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index d7f038d6d..a7053ed4d 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -32,7 +32,7 @@ ) -class ChannelWindow(BaseModel): +class Window(BaseModel): """ Custom class for Omero-channel window, related to OME-NGFF v0.4 @@ -53,7 +53,7 @@ class ChannelWindow(BaseModel): """TBD""" -class Channel(BaseModel): +class OmeroChannel(BaseModel): """ Custom class for Omero channels, related to OME-NGFF v0.4. @@ -74,7 +74,7 @@ class Channel(BaseModel): """TBD""" # From OME-NGFF v0.4 transitional metadata - window: Optional[ChannelWindow] + window: Optional[Window] """TBD""" color: Optional[str] """TBD""" @@ -97,7 +97,7 @@ class ChannelNotFoundError(ValueError): pass -def check_unique_wavelength_ids(channels: List[Channel]): +def check_unique_wavelength_ids(channels: List[OmeroChannel]): """ Check that the `wavelength_id` attributes of a channel list are unique """ @@ -148,8 +148,11 @@ def check_well_channel_labels(*, well_zarr_path: str) -> None: def get_channel_from_image_zarr( - *, image_zarr_path: str, label: str = None, wavelength_id: str = None -) -> Channel: + *, + image_zarr_path: str, + label: Optional[str] = None, + wavelength_id: Optional[str] = None, +) -> OmeroChannel: """ Extract a channel from OME-NGFF zarr attributes @@ -169,7 +172,7 @@ def get_channel_from_image_zarr( return channel -def get_omero_channel_list(*, image_zarr_path: str) -> List[Channel]: +def get_omero_channel_list(*, image_zarr_path: str) -> List[OmeroChannel]: """ Extract the list of channels from OME-NGFF zarr attributes @@ -178,13 +181,16 @@ def get_omero_channel_list(*, image_zarr_path: str) -> List[Channel]: """ group = zarr.open_group(image_zarr_path, mode="r+") channels_dicts = group.attrs["omero"]["channels"] - channels = [Channel(**c) for c in channels_dicts] + channels = [OmeroChannel(**c) for c in channels_dicts] return channels def get_channel_from_list( - *, channels: List[Channel], label: str = None, wavelength_id: str = None -) -> Channel: + *, + channels: List[OmeroChannel], + label: Optional[str] = None, + wavelength_id: Optional[str] = None, +) -> OmeroChannel: """ Find matching channel in a list @@ -245,9 +251,9 @@ def get_channel_from_list( def define_omero_channels( *, - channels: List[Channel], + channels: List[OmeroChannel], bit_depth: int, - label_prefix: str = None, + label_prefix: Optional[str] = None, ) -> List[Dict[str, Union[str, int, bool, Dict[str, int]]]]: """ Update a channel list to use it in the OMERO/channels metadata diff --git a/fractal_tasks_core/lib_regions_of_interest.py b/fractal_tasks_core/lib_regions_of_interest.py index 44d94a193..ed711094d 100644 --- a/fractal_tasks_core/lib_regions_of_interest.py +++ b/fractal_tasks_core/lib_regions_of_interest.py @@ -407,11 +407,11 @@ def is_ROI_table_valid(*, table_path: str, use_masks: bool) -> Optional[bool]: def load_region( - data_zyx: da.array, + data_zyx: da.Array, region: Tuple[slice, slice, slice], compute=True, return_as_3D=False, -) -> Union[da.array, np.array]: +) -> Union[da.Array, np.ndarray]: """ Load a region from a dask array diff --git a/fractal_tasks_core/tasks/_input_models.py b/fractal_tasks_core/tasks/_input_models.py new file mode 100644 index 000000000..f6f70de57 --- /dev/null +++ b/fractal_tasks_core/tasks/_input_models.py @@ -0,0 +1,101 @@ +""" +Copyright 2022 (C) + Friedrich Miescher Institute for Biomedical Research and + University of Zurich + + Original authors: + Tommaso Comparin + + This file is part of Fractal and was originally developed by eXact lab + S.r.l. under contract with Liberali Lab from the Friedrich + Miescher Institute for Biomedical Research and Pelkmans Lab from the + University of Zurich. + +Pydantic models for some task parameters +""" +from typing import Literal +from typing import Optional + +from pydantic import BaseModel +from pydantic import validator + + +class Channel(BaseModel): + """ + A channel which is specified by either ``wavelength_id`` or ``label``. + """ + + wavelength_id: Optional[str] = None + label: Optional[str] = None + + @validator("label", always=True) + def mutually_exclusive_channel_attributes(cls, v, values): + """ + Attributes ``label`` and ``wavelength_id`` are mutually exclusive. + """ + wavelength_id = values.get("wavelength_id") + label = v + if wavelength_id and v: + raise ValueError( + "`wavelength_id` and `label` cannot be both set " + f"(given {wavelength_id=} and {label=})." + ) + if wavelength_id is None and v is None: + raise ValueError( + "`wavelength_id` and `label` cannot be both `None`" + ) + return v + + +class NapariWorkflowsInput(BaseModel): + """ + A value of the ``input_specs`` argument in ``napari_workflows_wrapper``. + """ + + type: Literal["image", "label"] + label_name: Optional[str] + channel: Optional[Channel] + + @validator("label_name", always=True) + def label_name_is_present(cls, v, values): + _type = values.get("type") + if _type == "label" and not v: + raise ValueError( + f"Input item has type={_type} but label_name={v}." + ) + return v + + @validator("channel", always=True) + def channel_is_present(cls, v, values): + _type = values.get("type") + if _type == "image" and not v: + raise ValueError(f"Input item has type={_type} but channel={v}.") + return v + + +class NapariWorkflowsOutput(BaseModel): + """ + A value of the ``output_specs`` argument in ``napari_workflows_wrapper``. + """ + + type: Literal["label", "dataframe"] + label_name: Optional[str] = None + table_name: Optional[str] = None + + @validator("label_name", always=True) + def label_name_only_for_label_type(cls, v, values): + _type = values.get("type") + if (_type == "label" and (not v)) or (_type != "label" and v): + raise ValueError( + f"Output item has type={_type} but label_name={v}." + ) + return v + + @validator("table_name", always=True) + def table_name_only_for_dataframe_type(cls, v, values): + _type = values.get("type") + if (_type == "dataframe" and (not v)) or (_type != "dataframe" and v): + raise ValueError( + f"Output item has type={_type} but table_name={v}." + ) + return v diff --git a/fractal_tasks_core/tasks/_utils.py b/fractal_tasks_core/tasks/_utils.py index 1388fb8aa..5e408e6e2 100644 --- a/fractal_tasks_core/tasks/_utils.py +++ b/fractal_tasks_core/tasks/_utils.py @@ -24,6 +24,7 @@ from json import JSONEncoder from pathlib import Path from typing import Callable +from typing import Optional class TaskParameterEncoder(JSONEncoder): @@ -40,7 +41,7 @@ def default(self, value): def run_fractal_task( *, task_function: Callable, - logger_name: str = None, + logger_name: Optional[str] = None, ): """ Implement standard task interface and call task_function diff --git a/fractal_tasks_core/tasks/cellpose_segmentation.py b/fractal_tasks_core/tasks/cellpose_segmentation.py index 7486e4f11..7baac1994 100644 --- a/fractal_tasks_core/tasks/cellpose_segmentation.py +++ b/fractal_tasks_core/tasks/cellpose_segmentation.py @@ -38,6 +38,7 @@ import fractal_tasks_core from fractal_tasks_core.lib_channels import ChannelNotFoundError from fractal_tasks_core.lib_channels import get_channel_from_image_zarr +from fractal_tasks_core.lib_channels import OmeroChannel from fractal_tasks_core.lib_masked_loading import masked_loading_wrapper from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( @@ -52,6 +53,7 @@ from fractal_tasks_core.lib_ROI_overlaps import get_overlapping_pairs_3D from fractal_tasks_core.lib_zattrs_utils import extract_zyx_pixel_sizes from fractal_tasks_core.lib_zattrs_utils import rescale_datasets +from fractal_tasks_core.tasks._input_models import Channel logger = logging.getLogger(__name__) @@ -152,10 +154,8 @@ def cellpose_segmentation( metadata: Dict[str, Any], # Task-specific arguments level: int, - wavelength_id: Optional[str] = None, - channel_label: Optional[str] = None, - wavelength_id_c2: Optional[str] = None, - channel_label_c2: Optional[str] = None, + channel: Channel, + channel2: Optional[Channel] = None, input_ROI_table: str = "FOV_ROI_table", output_ROI_table: Optional[str] = None, output_label_name: Optional[str] = None, @@ -204,24 +204,14 @@ def cellpose_segmentation( managed by Fractal server) :param level: Pyramid level of the image to be segmented. Choose 0 to process at full resolution. - :param wavelength_id: Identifier of a channel based on the - wavelength (e.g. ``A01_C01``). If not ``None``, then - ``channel_label` must be ``None``. - :param channel_label: Identifier of a channel based on its label (e.g. - ``DAPI``). If not ``None``, then ``wavelength_id`` - must be ``None``. - :param wavelength_id_c2: Identifier of a second channel in the same format - as the first wavelength_id. If specified, cellpose - runs in dual channel mode. For dual channel - segmentation of cells, the first channel should - contain the membrane marker, the second channel - should contain the nuclear marker. - :param channel_label_c2: Identifier of a second channel in the same - format as the first wavelength_id. If specified, - cellpose runs in dual channel mode. For dual - channel segmentation of cells, the first channel - should contain the membrane marker, the second - channel should contain the nuclear marker. + :param channel: Primary channel for segmentation; requires either + ``wavelength_id`` (e.g. ``A01_C01``) or ``label`` (e.g. + ``DAPI``). + :param channel2: Second channel for segmentation (in the same format as + ``channel``). If specified, cellpose runs in dual channel + mode. For dual channel segmentation of cells, the first + channel should contain the membrane marker, the second + channel should contain the nuclear marker. :param input_ROI_table: Name of the ROI table over which the task loops to apply Cellpose segmentation. Example: "FOV_ROI_table" => loop over the field of @@ -295,15 +285,6 @@ def cellpose_segmentation( zarrurl = (in_path.resolve() / component).as_posix() logger.info(f"{zarrurl=}") - # Preliminary check - if (channel_label is None and wavelength_id is None) or ( - channel_label and wavelength_id - ): - raise ValueError( - f"One and only one of {channel_label=} and " - f"{wavelength_id=} arguments must be provided" - ) - # Preliminary checks on Cellpose model if pretrained_model is None: if model_type not in models.MODEL_NAMES: @@ -320,10 +301,10 @@ def cellpose_segmentation( # Find channel index try: - channel = get_channel_from_image_zarr( + tmp_channel: OmeroChannel = get_channel_from_image_zarr( image_zarr_path=zarrurl, - wavelength_id=wavelength_id, - label=channel_label, + wavelength_id=channel.wavelength_id, + label=channel.label, ) except ChannelNotFoundError as e: logger.warning( @@ -331,30 +312,29 @@ def cellpose_segmentation( f"Original error: {str(e)}" ) return {} - ind_channel = channel.index + ind_channel = tmp_channel.index # Find channel index for second channel, if one is provided - if wavelength_id_c2 or channel_label_c2: + if channel2: try: - channel_c2 = get_channel_from_image_zarr( + tmp_channel_c2: OmeroChannel = get_channel_from_image_zarr( image_zarr_path=zarrurl, - wavelength_id=wavelength_id_c2, - label=channel_label_c2, + wavelength_id=channel2.wavelength_id, + label=channel2.label, ) except ChannelNotFoundError as e: logger.warning( - f"Second channel with wavelength_id_c2:{wavelength_id_c2} and " - f"channel_label_c2: {channel_label_c2} not found, exit " - "from the task.\n" + f"Second channel with wavelength_id: {channel2.wavelength_id} " + f"and label: {channel2.label} not found, exit from the task.\n" f"Original error: {str(e)}" ) return {} - ind_channel_c2 = channel_c2.index + ind_channel_c2 = tmp_channel_c2.index # Set channel label if output_label_name is None: try: - channel_label = channel.label + channel_label = tmp_channel.label output_label_name = f"label_{channel_label}" except (KeyError, IndexError): output_label_name = f"label_{ind_channel}" @@ -362,7 +342,7 @@ def cellpose_segmentation( # Load ZYX data data_zyx = da.from_zarr(f"{zarrurl}/{level}")[ind_channel] logger.info(f"{data_zyx.shape=}") - if wavelength_id_c2 or channel_label_c2: + if channel2: data_zyx_c2 = da.from_zarr(f"{zarrurl}/{level}")[ind_channel_c2] logger.info(f"Second channel: {data_zyx_c2.shape=}") @@ -547,7 +527,7 @@ def cellpose_segmentation( logger.info("Total well shape/chunks:") logger.info(f"{data_zyx.shape}") logger.info(f"{data_zyx.chunks}") - if wavelength_id_c2 or channel_label_c2: + if channel2: logger.info("Dual channel input for cellpose model") logger.info(f"{data_zyx_c2.shape}") logger.info(f"{data_zyx_c2.chunks}") @@ -574,7 +554,7 @@ def cellpose_segmentation( logger.info(f"Now processing ROI {i_ROI+1}/{num_ROIs}") # Prepare single-channel or dual-channel input for cellpose - if wavelength_id_c2 or channel_label_c2: + if channel2: # Dual channel mode, first channel is the membrane channel img_1 = load_region( data_zyx, diff --git a/fractal_tasks_core/tasks/create_ome_zarr.py b/fractal_tasks_core/tasks/create_ome_zarr.py index 01e793e44..4942efe3b 100644 --- a/fractal_tasks_core/tasks/create_ome_zarr.py +++ b/fractal_tasks_core/tasks/create_ome_zarr.py @@ -27,10 +27,10 @@ from pydantic.decorator import validate_arguments import fractal_tasks_core -from fractal_tasks_core.lib_channels import Channel from fractal_tasks_core.lib_channels import check_unique_wavelength_ids from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels +from fractal_tasks_core.lib_channels import OmeroChannel from fractal_tasks_core.lib_glob import glob_with_multiple_patterns from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename @@ -54,7 +54,7 @@ def create_ome_zarr( metadata: Dict[str, Any], image_extension: str = "tif", image_glob_patterns: Optional[list[str]] = None, - allowed_channels: List[Channel], + allowed_channels: List[OmeroChannel], num_levels: int = 5, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -107,12 +107,10 @@ def create_ome_zarr( :param coarsening_xy: Linear coarsening factor between subsequent levels. If set to 2, level 1 is 2x downsampled, level 2 is 4x downsampled etc. - :param allowed_channels: A list of channel dictionaries, where each channel - must include the ``wavelength_id`` key and where - the corresponding values should be unique across - channels. - # TODO: improve after Channel input refactor - See issue 386 + :param allowed_channels: A list of ``OmeroChannel``s, where each channel + must include the ``wavelength_id`` attribute and + where the ``wavelength_id`` values must be unique + across the list. :param metadata_table: If equal to ``"mrf_mlf"``, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, the full path to a csv file containing diff --git a/fractal_tasks_core/tasks/create_ome_zarr_multiplex.py b/fractal_tasks_core/tasks/create_ome_zarr_multiplex.py index ff878fcf7..761e1e469 100644 --- a/fractal_tasks_core/tasks/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/tasks/create_ome_zarr_multiplex.py @@ -29,10 +29,10 @@ from pydantic.decorator import validate_arguments import fractal_tasks_core -from fractal_tasks_core.lib_channels import Channel from fractal_tasks_core.lib_channels import check_unique_wavelength_ids from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels +from fractal_tasks_core.lib_channels import OmeroChannel from fractal_tasks_core.lib_glob import glob_with_multiple_patterns from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename @@ -56,7 +56,7 @@ def create_ome_zarr_multiplex( metadata: Dict[str, Any], image_extension: str = "tif", image_glob_patterns: Optional[list[str]] = None, - allowed_channels: Dict[str, list[Channel]], + allowed_channels: Dict[str, list[OmeroChannel]], num_levels: int = 5, coarsening_xy: int = 2, metadata_table: Union[Literal["mrf_mlf"], Dict[str, str]] = "mrf_mlf", @@ -107,14 +107,11 @@ def create_ome_zarr_multiplex( :param coarsening_xy: Linear coarsening factor between subsequent levels. If set to 2, level 1 is 2x downsampled, level 2 is 4x downsampled etc. - :param allowed_channels: A dictionary of channel dictionaries, where each - channel must include the ``wavelength_id`` key - and where the corresponding values should be - unique across channels. - Values are the integers of the channel order, - i.e. ``"0"``, ``"1"`` etc. - # TODO: improve after Channel input refactor - https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/386 + :param allowed_channels: A dictionary of lists of ``OmeroChannel``s, where + each channel must include the ``wavelength_id`` + attribute and where the ``wavelength_id`` values + must be unique across each list. Dictionary keys + represent channel indices (``"0","1",..``). :param metadata_table: If equal to ``"mrf_mlf"``, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, a dictionary of key-value pairs like @@ -160,10 +157,10 @@ def create_ome_zarr_multiplex( # Preliminary checks on allowed_channels # Note that in metadata the keys of dictionary arguments should be # strings (and not integers), so that they can be read from a JSON file - for key, value in allowed_channels.items(): + for key, _channels in allowed_channels.items(): if not isinstance(key, str): raise ValueError(f"{allowed_channels=} has non-string keys") - check_unique_wavelength_ids(value) + check_unique_wavelength_ids(_channels) # Identify all plates and all channels, per input folders dict_acquisitions: Dict = {} diff --git a/fractal_tasks_core/tasks/illumination_correction.py b/fractal_tasks_core/tasks/illumination_correction.py index b076a9930..87783ec29 100644 --- a/fractal_tasks_core/tasks/illumination_correction.py +++ b/fractal_tasks_core/tasks/illumination_correction.py @@ -30,8 +30,8 @@ from pydantic.decorator import validate_arguments from skimage.io import imread -from fractal_tasks_core.lib_channels import Channel from fractal_tasks_core.lib_channels import get_omero_channel_list +from fractal_tasks_core.lib_channels import OmeroChannel from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( convert_ROI_table_to_indices, @@ -197,7 +197,7 @@ def illumination_correction( logger.info(f" {zarrurl_new=}") # Read channels from .zattrs - channels: list[Channel] = get_omero_channel_list( + channels: list[OmeroChannel] = get_omero_channel_list( image_zarr_path=zarrurl_old ) num_channels = len(channels) diff --git a/fractal_tasks_core/tasks/napari_workflows_wrapper.py b/fractal_tasks_core/tasks/napari_workflows_wrapper.py index ed452cf53..93fe6a362 100644 --- a/fractal_tasks_core/tasks/napari_workflows_wrapper.py +++ b/fractal_tasks_core/tasks/napari_workflows_wrapper.py @@ -42,6 +42,8 @@ from fractal_tasks_core.lib_upscale_array import upscale_array from fractal_tasks_core.lib_zattrs_utils import extract_zyx_pixel_sizes from fractal_tasks_core.lib_zattrs_utils import rescale_datasets +from fractal_tasks_core.tasks._input_models import NapariWorkflowsInput +from fractal_tasks_core.tasks._input_models import NapariWorkflowsOutput __OME_NGFF_VERSION__ = fractal_tasks_core.__OME_NGFF_VERSION__ @@ -68,8 +70,8 @@ def napari_workflows_wrapper( metadata: Dict[str, Any], # Task-specific arguments: workflow_file: str, - input_specs: Dict[str, Dict[str, str]], - output_specs: Dict[str, Dict[str, str]], + input_specs: Dict[str, NapariWorkflowsInput], + output_specs: Dict[str, NapariWorkflowsOutput], input_ROI_table: str = "FOV_ROI_table", level: int = 0, relabeling: bool = True, @@ -85,8 +87,8 @@ def napari_workflows_wrapper( # Examples of allowed entries for input_specs and output_specs input_specs = { - "in_1": {"type": "image", "wavelength_id": "A01_C02"}, - "in_2": {"type": "image", "channel_label": "DAPI"}, + "in_1": {"type": "image", "channel": {"wavelength_id": "A01_C02"}}, + "in_2": {"type": "image", "channel": {"label": "DAPI"}}, "in_3": {"type": "label", "label_name": "label_DAPI"}, } output_specs = { @@ -121,8 +123,8 @@ def napari_workflows_wrapper( (standard argument for Fractal tasks, managed by Fractal server) :param workflow_file: Absolute path to napari-workflows YAML file - :param input_specs: See examples above. TODO: Update after issue 404 - :param output_specs: See examples above. TODO: Update after issue 404 + :param input_specs: A dictionary of ``NapariWorkflowsInput`` values. + :param output_specs: A dictionary of ``NapariWorkflowsOutput`` values. :param input_ROI_table: Name of the ROI table over which the task loops to apply napari workflows. Example: "FOV_ROI_table" => loop over the field of @@ -161,8 +163,10 @@ def napari_workflows_wrapper( list_outputs = sorted(output_specs.keys()) # Characterization of workflow and scope restriction - input_types = [params["type"] for (name, params) in input_specs.items()] - output_types = [params["type"] for (name, params) in output_specs.items()] + input_types = [in_params.type for (name, in_params) in input_specs.items()] + output_types = [ + out_params.type for (name, out_params) in output_specs.items() + ] are_inputs_all_images = set(input_types) == {"image"} are_outputs_all_labels = set(output_types) == {"label"} are_outputs_all_dataframes = set(output_types) == {"dataframe"} @@ -192,7 +196,9 @@ def napari_workflows_wrapper( # Pre-processing of task inputs if len(input_paths) > 1: - raise NotImplementedError("We currently only support a single in_path") + raise NotImplementedError( + "We currently only support a single input path" + ) in_path = Path(input_paths[0]).as_posix() num_levels = metadata["num_levels"] coarsening_xy = metadata["coarsening_xy"] @@ -236,25 +242,19 @@ def napari_workflows_wrapper( # Input preparation: "image" type image_inputs = [ - (name, params) - for (name, params) in input_specs.items() - if params["type"] == "image" + (name, in_params) + for (name, in_params) in input_specs.items() + if in_params.type == "image" ] input_image_arrays = {} if image_inputs: img_array = da.from_zarr(f"{in_path}/{component}/{level}") # Loop over image inputs and assign corresponding channel of the image for (name, params) in image_inputs: - if "wavelength_id" in params and "channel_label" in params: - raise ValueError( - "One and only one among channel_label and wavelength_id" - f" attributes must be provided, but input {name} in " - f"input_specs has {params=}." - ) channel = get_channel_from_image_zarr( image_zarr_path=f"{in_path}/{component}", - wavelength_id=params.get("wavelength_id", None), - label=params.get("channel_label", None), + wavelength_id=params.channel.wavelength_id, + label=params.channel.label, ) channel_index = channel.index input_image_arrays[name] = img_array[channel_index] @@ -286,9 +286,9 @@ def napari_workflows_wrapper( # Input preparation: "label" type label_inputs = [ - (name, params) - for (name, params) in input_specs.items() - if params["type"] == "label" + (name, in_params) + for (name, in_params) in input_specs.items() + if in_params.type == "label" ] if label_inputs: # Set target_shape for upscaling labels @@ -304,7 +304,7 @@ def napari_workflows_wrapper( # Loop over label inputs and load corresponding (upscaled) image input_label_arrays = {} for (name, params) in label_inputs: - label_name = params["label_name"] + label_name = params.label_name label_array_raw = da.from_zarr( f"{in_path}/{component}/labels/{label_name}/{level}" ) @@ -360,9 +360,9 @@ def napari_workflows_wrapper( # Output preparation: "label" type label_outputs = [ - (name, params) - for (name, params) in output_specs.items() - if params["type"] == "label" + (name, out_params) + for (name, out_params) in output_specs.items() + if out_params.type == "label" ] if label_outputs: # Preliminary scope checks @@ -388,7 +388,7 @@ def napari_workflows_wrapper( elif label_inputs: reference_array = list(input_label_arrays.values())[0] # Re-load pixel size, matching to the correct level - input_label_name = label_inputs[0][1]["label_name"] + input_label_name = label_inputs[0][1].label_name zattrs_file = ( f"{in_path}/{component}/labels/{input_label_name}/.zattrs" ) @@ -434,7 +434,7 @@ def napari_workflows_wrapper( logger.info(f"{label_chunksize=}") # Create labels zarr group and combine existing/new labels in .zattrs - new_labels = [params["label_name"] for (name, params) in label_outputs] + new_labels = [params.label_name for (name, params) in label_outputs] zarrurl = f"{in_path}/{component}" try: with open(f"{zarrurl}/labels/.zattrs", "r") as f_zattrs: @@ -454,8 +454,8 @@ def napari_workflows_wrapper( # Loop over label outputs and (1) set zattrs, (2) create zarr group output_label_zarr_groups: Dict[str, Any] = {} - for (name, params) in label_outputs: - label_name = params["label_name"] + for (name, out_params) in label_outputs: + label_name = out_params.label_name # (1a) Rescale OME-NGFF datasets (relevant for level>0) if not multiscales[0]["axes"][0]["name"] == "c": @@ -500,19 +500,19 @@ def napari_workflows_wrapper( dimension_separator="/", ) output_label_zarr_groups[name] = mask_zarr - logger.info(f"Prepared output with {name=} and {params=}") + logger.info(f"Prepared output with {name=} and {out_params=}") logger.info(f"{output_label_zarr_groups=}") # Output preparation: "dataframe" type dataframe_outputs = [ - (name, params) - for (name, params) in output_specs.items() - if params["type"] == "dataframe" + (name, out_params) + for (name, out_params) in output_specs.items() + if out_params.type == "dataframe" ] output_dataframe_lists: Dict[str, List] = {} - for (name, params) in dataframe_outputs: + for (name, out_params) in dataframe_outputs: output_dataframe_lists[name] = [] - logger.info(f"Prepared output with {name=} and {params=}") + logger.info(f"Prepared output with {name=} and {out_params=}") logger.info(f"{output_dataframe_lists=}") ##### @@ -528,7 +528,7 @@ def napari_workflows_wrapper( # Set inputs for input_name in input_specs.keys(): - input_type = input_specs[input_name]["type"] + input_type = input_specs[input_name].type if input_type == "image": wf.set( @@ -557,7 +557,7 @@ def napari_workflows_wrapper( # Iterate first over dataframe outputs (to use the correct # max_label_for_relabeling, if needed) for ind_output, output_name in enumerate(list_outputs): - if output_specs[output_name]["type"] != "dataframe": + if output_specs[output_name].type != "dataframe": continue df = outputs[ind_output] if relabeling: @@ -573,7 +573,7 @@ def napari_workflows_wrapper( # After all dataframe outputs, iterate over label outputs (which # actually can be only 0 or 1) for ind_output, output_name in enumerate(list_outputs): - if output_specs[output_name]["type"] != "label": + if output_specs[output_name].type != "label": continue mask = outputs[ind_output] @@ -623,8 +623,8 @@ def napari_workflows_wrapper( # Output handling: "dataframe" type (for each output, concatenate ROI # dataframes, clean up, and store in a AnnData table on-disk) - for (name, params) in dataframe_outputs: - table_name = params["table_name"] + for (name, out_params) in dataframe_outputs: + table_name = out_params.table_name # Concatenate all FOV dataframes list_dfs = output_dataframe_lists[name] if len(list_dfs) == 0: @@ -658,8 +658,8 @@ def napari_workflows_wrapper( # Output handling: "label" type (for each output, build and write to disk # pyramid of coarser levels) - for (name, params) in label_outputs: - label_name = params["label_name"] + for (name, out_params) in label_outputs: + label_name = out_params.label_name build_pyramid( zarrurl=f"{zarrurl}/labels/{label_name}", overwrite=False, diff --git a/fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py b/fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py index 263d6cf07..7575ea06c 100644 --- a/fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py +++ b/fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py @@ -27,8 +27,8 @@ from dask.array.image import imread from pydantic.decorator import validate_arguments -from fractal_tasks_core.lib_channels import Channel from fractal_tasks_core.lib_channels import get_omero_channel_list +from fractal_tasks_core.lib_channels import OmeroChannel from fractal_tasks_core.lib_glob import glob_with_multiple_patterns from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_pyramid_creation import build_pyramid @@ -137,7 +137,9 @@ def yokogawa_to_ome_zarr( image_extension = parameters["image_extension"] image_glob_patterns = parameters["image_glob_patterns"] - channels: list[Channel] = get_omero_channel_list(image_zarr_path=zarrurl) + channels: list[OmeroChannel] = get_omero_channel_list( + image_zarr_path=zarrurl + ) wavelength_ids = [c.wavelength_id for c in channels] in_path = Path(original_path_list[0]) @@ -195,11 +197,11 @@ def yokogawa_to_ome_zarr( patterns = [f"*_{well_ID}_*{A}*{C}*.{image_extension}"] if image_glob_patterns: patterns.extend(image_glob_patterns) - filenames = glob_with_multiple_patterns( + filenames_set = glob_with_multiple_patterns( folder=str(in_path), patterns=patterns, ) - filenames = sorted(list(filenames), key=sort_fun) + filenames = sorted(list(filenames_set), key=sort_fun) if len(filenames) == 0: raise Exception( "Error in yokogawa_to_ome_zarr: len(filenames)=0.\n" diff --git a/poetry.lock b/poetry.lock index 0b53105c6..e29168461 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2215,6 +2215,21 @@ files = [ {file = "lit-16.0.5.post0.tar.gz", hash = "sha256:71745d9e58dad3717735d27e2a9cca0e9ca6861d067da73c307e02fd38c98479"}, ] +[[package]] +name = "livereload" +version = "2.6.3" +description = "Python LiveReload is an awesome tool for web developers" +optional = false +python-versions = "*" +files = [ + {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, + {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, +] + +[package.dependencies] +six = "*" +tornado = {version = "*", markers = "python_version > \"2.7\""} + [[package]] name = "llvmlite" version = "0.39.1" @@ -2718,6 +2733,52 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] +[[package]] +name = "mypy" +version = "1.3.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mypy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eb485cea53f4f5284e5baf92902cd0088b24984f4209e25981cc359d64448d"}, + {file = "mypy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c99c3ecf223cf2952638da9cd82793d8f3c0c5fa8b6ae2b2d9ed1e1ff51ba85"}, + {file = "mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:550a8b3a19bb6589679a7c3c31f64312e7ff482a816c96e0cecec9ad3a7564dd"}, + {file = "mypy-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cbc07246253b9e3d7d74c9ff948cd0fd7a71afcc2b77c7f0a59c26e9395cb152"}, + {file = "mypy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a22435632710a4fcf8acf86cbd0d69f68ac389a3892cb23fbad176d1cddaf228"}, + {file = "mypy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e33bb8b2613614a33dff70565f4c803f889ebd2f859466e42b46e1df76018dd"}, + {file = "mypy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d23370d2a6b7a71dc65d1266f9a34e4cde9e8e21511322415db4b26f46f6b8c"}, + {file = "mypy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658fe7b674769a0770d4b26cb4d6f005e88a442fe82446f020be8e5f5efb2fae"}, + {file = "mypy-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d29e324cdda61daaec2336c42512e59c7c375340bd202efa1fe0f7b8f8ca"}, + {file = "mypy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d0b6c62206e04061e27009481cb0ec966f7d6172b5b936f3ead3d74f29fe3dcf"}, + {file = "mypy-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:76ec771e2342f1b558c36d49900dfe81d140361dd0d2df6cd71b3db1be155409"}, + {file = "mypy-1.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc95f8386314272bbc817026f8ce8f4f0d2ef7ae44f947c4664efac9adec929"}, + {file = "mypy-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:faff86aa10c1aa4a10e1a301de160f3d8fc8703b88c7e98de46b531ff1276a9a"}, + {file = "mypy-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8c5979d0deb27e0f4479bee18ea0f83732a893e81b78e62e2dda3e7e518c92ee"}, + {file = "mypy-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c5d2cc54175bab47011b09688b418db71403aefad07cbcd62d44010543fc143f"}, + {file = "mypy-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:87df44954c31d86df96c8bd6e80dfcd773473e877ac6176a8e29898bfb3501cb"}, + {file = "mypy-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473117e310febe632ddf10e745a355714e771ffe534f06db40702775056614c4"}, + {file = "mypy-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:74bc9b6e0e79808bf8678d7678b2ae3736ea72d56eede3820bd3849823e7f305"}, + {file = "mypy-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:44797d031a41516fcf5cbfa652265bb994e53e51994c1bd649ffcd0c3a7eccbf"}, + {file = "mypy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ddae0f39ca146972ff6bb4399f3b2943884a774b8771ea0a8f50e971f5ea5ba8"}, + {file = "mypy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c4c42c60a8103ead4c1c060ac3cdd3ff01e18fddce6f1016e08939647a0e703"}, + {file = "mypy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86c2c6852f62f8f2b24cb7a613ebe8e0c7dc1402c61d36a609174f63e0ff017"}, + {file = "mypy-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f9dca1e257d4cc129517779226753dbefb4f2266c4eaad610fc15c6a7e14283e"}, + {file = "mypy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:95d8d31a7713510685b05fbb18d6ac287a56c8f6554d88c19e73f724a445448a"}, + {file = "mypy-1.3.0-py3-none-any.whl", hash = "sha256:a8763e72d5d9574d45ce5881962bc8e9046bf7b375b0abf031f3e6811732a897"}, + {file = "mypy-1.3.0.tar.gz", hash = "sha256:e1f4d16e296f5135624b34e8fb741eb0eadedca90862405b1f1fde2040b9bd11"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -4920,6 +4981,25 @@ docs = ["sphinxcontrib-websupport"] lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] +[[package]] +name = "sphinx-autobuild" +version = "2021.3.14" +description = "Rebuild Sphinx documentation on changes, with live-reload in the browser." +optional = false +python-versions = ">=3.6" +files = [ + {file = "sphinx-autobuild-2021.3.14.tar.gz", hash = "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05"}, + {file = "sphinx_autobuild-2021.3.14-py3-none-any.whl", hash = "sha256:8fe8cbfdb75db04475232f05187c776f46f6e9e04cacf1e49ce81bdac649ccac"}, +] + +[package.dependencies] +colorama = "*" +livereload = "*" +sphinx = "*" + +[package.extras] +test = ["pytest", "pytest-cov"] + [[package]] name = "sphinx-autodoc-defaultargs" version = "0.1.2" @@ -5849,4 +5929,4 @@ fractal-tasks = ["Pillow", "cellpose", "imageio-ffmpeg", "llvmlite", "napari-seg [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d8b234720f36584a1a6cc49bd2e514acacb5fac9c8cacac48483bdce146522a0" +content-hash = "02c40f6cb82d55c64a5f111aa3c107639bed3001ad6376fb6e28c7c2bdddbcdf" diff --git a/pyproject.toml b/pyproject.toml index 6e7e4af58..6c091bd10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ jsonschema = "^4.16.0" deptry = "^0.6.1" napari-ome-zarr = "^0.5.2" pytest-pretty = "^1.1.0" +mypy = "^1.3.0" [tool.poetry.group.docs.dependencies] sphinx = "^5.3.0" @@ -67,6 +68,7 @@ sphinx-autodoc-typehints = "^1.19.5" autodocsumm = "^0.2.9" sphinx-autodoc-defaultargs = "^0.1.2" myst-parser = "^0.18.1" +sphinx-autobuild = "^2021.3.14" [build-system] diff --git a/tests/tasks/test_unit_input_models.py b/tests/tasks/test_unit_input_models.py new file mode 100644 index 000000000..2c2cca9ad --- /dev/null +++ b/tests/tasks/test_unit_input_models.py @@ -0,0 +1,117 @@ +import pytest +from devtools import debug + +from fractal_tasks_core.tasks._input_models import Channel +from fractal_tasks_core.tasks._input_models import ( + NapariWorkflowsInput, +) +from fractal_tasks_core.tasks._input_models import ( + NapariWorkflowsOutput, +) + + +def test_Channel(): + + # Valid + + c = Channel(wavelength_id="wavelength_id") + debug(c) + assert c.wavelength_id + assert not c.label + + c = Channel(label="label") + debug(c) + assert not c.wavelength_id + assert c.label + + # Invalid + + with pytest.raises(ValueError) as e: + Channel() + debug(e.value) + + with pytest.raises(ValueError) as e: + Channel(label="label", wavelength_id="wavelength_id") + debug(e.value) + + +def test_NapariWorkflowsInput(): + + # Invalid + + with pytest.raises(ValueError) as e: + NapariWorkflowsInput(type="invalid") + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsInput(type="image") + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsInput(type="label") + debug(e.value) + + # Valid + + spec = NapariWorkflowsInput(type="label", label_name="name") + assert spec.type + assert spec.label_name + assert not spec.channel + + spec = NapariWorkflowsInput(type="image", channel=dict(label="something")) + assert spec.type + assert not spec.label_name + assert spec.channel + + +def test_NapariWorkflowsOutput(): + + # Invalid + + with pytest.raises(ValueError) as e: + NapariWorkflowsOutput(type="invalid") + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsOutput( + type="label", + table_name="something", + ) + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsOutput( + type="label", + ) + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsOutput( + type="dataframe", + label_name="something", + ) + debug(e.value) + + with pytest.raises(ValueError) as e: + NapariWorkflowsOutput( + type="dataframe", + ) + debug(e.value) + + # Valid + + specs = NapariWorkflowsOutput( + type="label", + label_name="label_DAPI", + ) + debug(specs) + assert specs.type + assert specs.label_name + + specs = NapariWorkflowsOutput( + type="dataframe", + table_name="table_DAPI", + ) + debug(specs) + assert specs.type + assert specs.table_name diff --git a/tests/tasks/test_unit_napari_workflows_wrapper.py b/tests/tasks/test_unit_napari_workflows_wrapper.py index 9abdd6e75..cec2a433c 100644 --- a/tests/tasks/test_unit_napari_workflows_wrapper.py +++ b/tests/tasks/test_unit_napari_workflows_wrapper.py @@ -38,7 +38,9 @@ def test_input_specs(tmp_path, testdata_path): def test_output_specs(tmp_path, testdata_path, caplog): """ - WHEN calling napari_workflows_wrapper with invalid output_specs + WHEN + calling napari_workflows_wrapper with a mismatch between wf.leafs and + output_specs THEN raise a Warning """ caplog.set_level(logging.WARNING) @@ -48,9 +50,12 @@ def test_output_specs(tmp_path, testdata_path, caplog): testdata_path / "napari_workflows/wf_5-labeling_only.yaml" ) input_specs = { - "input_image": {"type": "image", "wavelength_id": "A01_C01"} + "input_image": { + "type": "image", + "channel": {"wavelength_id": "A01_C01"}, + } } - output_specs = {"asd": {"asd": "asd"}} + output_specs = {"some_output": {"type": "label", "label_name": "xxx"}} try: napari_workflows_wrapper( @@ -64,10 +69,11 @@ def test_output_specs(tmp_path, testdata_path, caplog): input_ROI_table="FOV_ROI_table", ) except Exception as e: - # The task will now fail for some other reason (its arguments are not - # valid), but we only care about the warning + # The task will fail, but we only care about the warning debug(e) + debug(caplog.text) + assert "WARNING" in caplog.text assert "Some item of wf.leafs" in caplog.text assert "is not part of output_specs" in caplog.text @@ -83,8 +89,14 @@ def test_level_setting_in_non_labeling_worfklow(tmp_path, testdata_path): # napari-workflows workflow_file = str(testdata_path / "napari_workflows/wf_3.yaml") input_specs = { - "slice_img": {"type": "image", "wavelength_id": "A01_C01"}, - "slice_img_c2": {"type": "image", "wavelength_id": "A01_C01"}, + "slice_img": { + "type": "image", + "channel": {"wavelength_id": "A01_C01"}, + }, + "slice_img_c2": { + "type": "image", + "channel": {"wavelength_id": "A01_C01"}, + }, } output_specs = { "Result of Expand labels (scikit-image, nsbatwm)": { diff --git a/tests/tasks/test_workflows_cellpose_segmentation.py b/tests/tasks/test_workflows_cellpose_segmentation.py index 1dde68b62..378f45378 100644 --- a/tests/tasks/test_workflows_cellpose_segmentation.py +++ b/tests/tasks/test_workflows_cellpose_segmentation.py @@ -170,14 +170,14 @@ def test_failures( # Attempt 1 cellpose_segmentation( **kwargs, - wavelength_id="invalid_wavelength_id", + channel=dict(wavelength_id="invalid_wavelength_id"), ) assert "ChannelNotFoundError" in caplog.records[0].msg # Attempt 2 cellpose_segmentation( **kwargs, - channel_label="invalid_channel_name", + channel=dict(label="invalid_channel_name"), ) assert "ChannelNotFoundError" in caplog.records[0].msg assert "ChannelNotFoundError" in caplog.records[1].msg @@ -186,8 +186,10 @@ def test_failures( with pytest.raises(ValueError): cellpose_segmentation( **kwargs, - wavelength_id="A01_C01", - channel_label="invalid_channel_name", + channel=dict( + wavelength_id="A01_C01", + label="invalid_channel_name", + ), ) @@ -229,7 +231,7 @@ def test_workflow_with_per_FOV_labeling( output_path=str(zarr_path), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=3, relabeling=True, diameter_level0=80.0, @@ -289,8 +291,8 @@ def test_workflow_with_multi_channel_input( output_path=str(zarr_path), metadata=metadata, component=component, - wavelength_id="A01_C01", - wavelength_id_c2="A01_C01", + channel=dict(wavelength_id="A01_C01"), + channel2=dict(wavelength_id="A01_C01"), level=3, relabeling=True, diameter_level0=80.0, @@ -346,7 +348,7 @@ def test_workflow_with_per_FOV_labeling_2D( output_path=str(zarr_path_mip), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=2, relabeling=True, diameter_level0=80.0, @@ -438,7 +440,7 @@ def test_workflow_with_per_well_labeling_2D( output_path=str(zarr_path_mip), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=2, input_ROI_table="well_ROI_table", relabeling=True, @@ -499,7 +501,7 @@ def test_workflow_bounding_box( output_path=str(zarr_path), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=3, relabeling=True, diameter_level0=80.0, @@ -556,7 +558,7 @@ def test_workflow_bounding_box_with_overlap( output_path=str(zarr_path), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=3, relabeling=True, diameter_level0=80.0, @@ -598,7 +600,7 @@ def test_workflow_with_per_FOV_labeling_via_script( output_path=str(zarr_path), metadata=metadata, component=metadata["image"][0], - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=4, relabeling=True, diameter_level0=80.0, @@ -672,7 +674,7 @@ def test_workflow_with_per_FOV_labeling_with_empty_FOV_table( metadata=metadata, component=component, input_ROI_table=TABLE_NAME, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=3, relabeling=True, diameter_level0=80.0, @@ -735,7 +737,7 @@ def test_CYX_input( output_path=str(zarr_path_mip), metadata=metadata, component=component, - wavelength_id="A01_C01", + channel=dict(wavelength_id="A01_C01"), level=0, relabeling=True, diameter_level0=80.0, diff --git a/tests/tasks/test_workflows_napari_workflows.py b/tests/tasks/test_workflows_napari_workflows.py index fd6e7c4e8..7d0231e0b 100644 --- a/tests/tasks/test_workflows_napari_workflows.py +++ b/tests/tasks/test_workflows_napari_workflows.py @@ -51,7 +51,7 @@ def test_napari_workflow( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "wavelength_id": "A01_C01"}, + "input": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -76,7 +76,7 @@ def test_napari_workflow( # Second napari-workflows task (measurement) workflow_file = str(testdata_path / "napari_workflows/wf_4.yaml") input_specs = { - "dapi_img": {"type": "image", "wavelength_id": "A01_C01"}, + "dapi_img": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, "dapi_label_img": {"type": "label", "label_name": "label_DAPI"}, } output_specs = { @@ -143,7 +143,7 @@ def test_napari_worfklow_label_input_only( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "wavelength_id": "A01_C01"}, + "input": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -209,13 +209,15 @@ def test_napari_worfklow_label_input_only( TABLE_NAME = "measurement_DAPI" # 1. Labeling-only workflow, from images to labels. workflow_file_name = "wf_relab_1-labeling_only.yaml" -input_specs = dict(input_image={"type": "image", "wavelength_id": "A01_C01"}) +input_specs = dict( + input_image={"type": "image", "channel": {"wavelength_id": "A01_C01"}} +) output_specs = dict(output_label={"type": "label", "label_name": LABEL_NAME}) RELABELING_CASE_1: List = [workflow_file_name, input_specs, output_specs] # 2. Measurement-only workflow, from images+labels to dataframes. workflow_file_name = "wf_relab_2-measurement_only.yaml" input_specs = dict( - input_image={"type": "image", "wavelength_id": "A01_C01"}, + input_image={"type": "image", "channel": {"wavelength_id": "A01_C01"}}, input_label={"type": "label", "label_name": LABEL_NAME}, ) output_specs = dict( @@ -224,7 +226,9 @@ def test_napari_worfklow_label_input_only( RELABELING_CASE_2: List = [workflow_file_name, input_specs, output_specs] # 3. Mixed labeling/measurement workflow. workflow_file_name = "wf_relab_3-labeling_and_measurement.yaml" -input_specs = dict(input_image={"type": "image", "wavelength_id": "A01_C01"}) +input_specs = dict( + input_image={"type": "image", "channel": {"wavelength_id": "A01_C01"}} +) output_specs = dict( output_label={"type": "label", "label_name": LABEL_NAME}, output_dataframe={"type": "dataframe", "table_name": TABLE_NAME}, @@ -406,7 +410,10 @@ def test_expected_dimensions( testdata_path / "napari_workflows/wf_5-labeling_only.yaml" ) input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input_image": {"type": "image", "wavelength_id": "A01_C01"}, + "input_image": { + "type": "image", + "channel": {"wavelength_id": "A01_C01"}, + }, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "output_label": { @@ -464,7 +471,7 @@ def test_napari_workflow_empty_input_ROI_table( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "wavelength_id": "A01_C01"}, + "input": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -489,7 +496,7 @@ def test_napari_workflow_empty_input_ROI_table( # Second napari-workflows task (measurement) workflow_file = str(testdata_path / "napari_workflows/wf_4.yaml") input_specs = { - "dapi_img": {"type": "image", "wavelength_id": "A01_C01"}, + "dapi_img": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, "dapi_label_img": {"type": "label", "label_name": "label_DAPI"}, } output_specs = { @@ -556,7 +563,7 @@ def test_napari_workflow_CYX( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "wavelength_id": "A01_C01"}, + "input": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -582,7 +589,7 @@ def test_napari_workflow_CYX( # Second napari-workflows task (measurement) workflow_file = str(testdata_path / "napari_workflows/wf_4.yaml") input_specs = { - "dapi_img": {"type": "image", "wavelength_id": "A01_C01"}, + "dapi_img": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, "dapi_label_img": {"type": "label", "label_name": "label_DAPI"}, } output_specs = { @@ -657,7 +664,7 @@ def test_napari_workflow_CYX_wrong_dimensions( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "wavelength_id": "A01_C01"}, + "input": {"type": "image", "channel": {"wavelength_id": "A01_C01"}}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { diff --git a/tests/test_unit_channels.py b/tests/test_unit_channels.py index 8190fe36c..e2be21c42 100644 --- a/tests/test_unit_channels.py +++ b/tests/test_unit_channels.py @@ -5,17 +5,17 @@ import pytest from devtools import debug -from fractal_tasks_core.lib_channels import Channel from fractal_tasks_core.lib_channels import define_omero_channels from fractal_tasks_core.lib_channels import get_channel_from_list +from fractal_tasks_core.lib_channels import OmeroChannel def test_get_channel_from_list(testdata_path: Path): - # Read JSON data and cast into `Channel`s + # Read JSON data and cast into `OmeroChannel`s with (testdata_path / "omero/channels_list.json").open("r") as f: omero_channels_dict = json.load(f) - omero_channels = [Channel(**c) for c in omero_channels_dict] + omero_channels = [OmeroChannel(**c) for c in omero_channels_dict] debug(omero_channels) # Extract a channel from a list / case 1 @@ -64,16 +64,16 @@ def omero_channel_schema(): def test_define_omero_channels(testdata_path: Path, omero_channel_schema): """ - GIVEN a list of our custom `Channel` objects + GIVEN a list of our custom `OmeroChannel` objects WHEN calling `define_omero_channels` THEN the output channel dictionaries are valid Omero channels according to the OME-NGFF schema """ - # Read JSON data and cast into `Channel`s + # Read JSON data and cast into `OmeroChannel`s with (testdata_path / "omero/channels_list.json").open("r") as f: omero_channels_dict = json.load(f) - omero_channels = [Channel(**c) for c in omero_channels_dict] + omero_channels = [OmeroChannel(**c) for c in omero_channels_dict] debug(omero_channels) # Call define_omero_channels