Introduce import-ome-zarr task

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 # Unreleased
 
+* Tasks:
+    * Make `maximum_intensity_projection` task not depend on ROI tables (\#557).
+    * New task and helper functions:
+        * Introduce `import_ome_zarr` task (\#557).
+        * Introduce `get_single_image_ROI` and `get_image_grid_ROIs` (\#557)
+        * Introduce `detect_ome_ngff_type` (\#557).
 * Dependencies:
   * Restrict `Pillow` version to `<10.1` (\#571).
 * Testing:

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -1184,6 +1184,84 @@
       },
       "docs_info": "Apply registration to images by using a registered ROI table\n\nThis task consists of 4 parts:\n\n1. Mask all regions in images that are not available in the\nregistered ROI table and store each cycle aligned to the\nreference_cycle (by looping over ROIs).\n2. Do the same for all label images.\n3. Copy all tables from the non-aligned image to the aligned image\n(currently only works well if the only tables are well & FOV ROI tables\n(registered and original). Not implemented for measurement tables and\nother ROI tables).\n4. Clean up: Delete the old, non-aligned image and rename the new,\naligned image to take over its place.\n\nParallelization level: image",
       "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/apply_registration_to_image/#fractal_tasks_core.tasks.apply_registration_to_image.apply_registration_to_image"
+    },
+    {
+      "name": "Import OME-Zarr",
+      "executable": "tasks/import_ome_zarr.py",
+      "input_type": "zarr",
+      "output_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000
+      },
+      "args_schema": {
+        "title": "ImportOmeZarr",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "A length-one list with the parent folder of the OME-Zarr to be imported; e.g. `input_paths=[\"/somewhere\"]`, if the OME-Zarr path is `/somewhere/array.zarr`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Not used in this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "Not used in this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "zarr_name": {
+            "title": "Zarr Name",
+            "type": "string",
+            "description": "The OME-Zarr name, without its parent folder; e.g. `zarr_name=\"array.zarr\"`, if the OME-Zarr path is `/somewhere/array.zarr`."
+          },
+          "add_image_ROI_table": {
+            "title": "Add Image Roi Table",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether to add a `image_ROI_table` table to each image, with a single ROI covering the whole image."
+          },
+          "add_grid_ROI_table": {
+            "title": "Add Grid Roi Table",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether to add a `grid_ROI_table` table to each image, with the image split into a rectangular grid of ROIs."
+          },
+          "grid_y_shape": {
+            "title": "Grid Y Shape",
+            "default": 2,
+            "type": "integer",
+            "description": "Y shape of the ROI grid in `grid_ROI_table`."
+          },
+          "grid_x_shape": {
+            "title": "Grid X Shape",
+            "default": 2,
+            "type": "integer",
+            "description": "X shape of the ROI grid in `grid_ROI_table`."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "Whether new ROI tables (added when `add_image_ROI_table` and/or `add_grid_ROI_table` are `True`) can overwite existing ones."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "metadata",
+          "zarr_name"
+        ],
+        "additionalProperties": false
+      },
+      "docs_info": "Import an OME-Zarr into Fractal.\n\nThe current version of this task:\n\n1. Creates the appropriate components-related metadata, needed for\n   processing an existing OME-Zarr through Fractal.\n2. Optionally adds new ROI tables to the existing OME-Zarr.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/import_ome_zarr/#fractal_tasks_core.tasks.import_ome_zarr.import_ome_zarr"
     }
   ]
 }

fractal_tasks_core/lib_ngff.py

@@ -12,6 +12,9 @@
 from pydantic import validator
 
 
+logger = logging.getLogger(__name__)
+
+
 class Window(BaseModel):
     """
     Model for `Channel.window`.
@@ -417,3 +420,31 @@ def load_NgffWellMeta(zarr_path: str) -> NgffWellMeta:
             f"Original error:\n{str(e)}"
         )
         raise e
+
+
+def detect_ome_ngff_type(group: zarr.Group) -> str:
+    """
+    Given a Zarr group, find whether it is an OME-NGFF plate, well or image.
+
+    Args:
+        group: Zarr group
+
+    Returns:
+        The detected OME-NGFF type (`plate`, `well` or `image`).
+    """
+    attrs = group.attrs.asdict()
+    if "plate" in attrs.keys():
+        ngff_type = "plate"
+    elif "well" in attrs.keys():
+        ngff_type = "well"
+    elif "multiscales" in attrs.keys():
+        ngff_type = "image"
+    else:
+        error_msg = (
+            "Zarr group at cannot be identified as one "
+            "of OME-NGFF plate/well/image groups."
+        )
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+    logger.info(f"Zarr group identified as OME-NGFF {ngff_type}.")
+    return ngff_type

fractal_tasks_core/lib_pyramid_creation.py

@@ -12,6 +12,7 @@
 """
 Construct and write pyramid of lower-resolution levels.
 """
+import logging
 import pathlib
 from typing import Callable
 from typing import Optional
@@ -21,6 +22,8 @@
 import dask.array as da
 import numpy as np
 
+logger = logging.getLogger(__name__)
+
 
 def build_pyramid(
     *,
@@ -50,9 +53,11 @@ def build_pyramid(
     # Clean up zarrurl
     zarrurl = str(pathlib.Path(zarrurl))  # FIXME
     zarrurl_highres = f"{zarrurl}/0"
+    logger.info(f"[build_pyramid] High-resolution path: {zarrurl_highres}")
 
     # Lazily load highest-resolution data
     data_highres = da.from_zarr(zarrurl_highres)
+    logger.info(f"[build_pyramid] High-resolution data: {str(data_highres)}")
 
     # Check the number of axes and identify YX dimensions
     ndims = len(data_highres.shape)
@@ -88,6 +93,10 @@ def build_pyramid(
             newlevel_rechunked = newlevel
         else:
             newlevel_rechunked = newlevel.rechunk(chunksize)
+        logger.info(
+            f"[build_pyramid] Level {ind_level} data: "
+            f"{str(newlevel_rechunked)}"
+        )
 
         # Write zarr and store output (useful to construct next level)
         previous_level = newlevel_rechunked.to_zarr(

fractal_tasks_core/lib_regions_of_interest.py

@@ -13,6 +13,7 @@
 Functions to handle regions of interests (via pandas and AnnData).
 """
 import logging
+import math
 from typing import Optional
 from typing import Sequence
 from typing import Union
@@ -606,3 +607,122 @@ def is_standard_roi_table(table: str) -> bool:
         return True
     else:
         return False
+
+
+def get_single_image_ROI(
+    array_shape: tuple[int, int, int],
+    pixels_ZYX: list[float],
+) -> ad.AnnData:
+    """
+    Produce a table with a single ROI that covers the whole array
+
+    TODO: make this flexible with respect to the presence/absence of Z.
+
+    Args:
+        array_shape: ZYX shape of the image array.
+        pixels_ZYX: ZYX pixel sizes in micrometers.
+
+    Returns:
+        An `AnnData` table with a single ROI.
+    """
+    shape_z, shape_y, shape_x = array_shape[-3:]
+    ROI_table = ad.AnnData(
+        X=np.array(
+            [
+                [
+                    0.0,
+                    0.0,
+                    0.0,
+                    shape_x * pixels_ZYX[2],
+                    shape_y * pixels_ZYX[1],
+                    shape_z * pixels_ZYX[0],
+                    0.0,
+                    0.0,
+                ],
+            ],
+            dtype=np.float32,
+        )
+    )
+    ROI_table.obs_names = ["image_1"]
+    ROI_table.var_names = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+    ]
+    return ROI_table
+
+
+def get_image_grid_ROIs(
+    array_shape: tuple[int, int, int],
+    pixels_ZYX: list[float],
+    grid_YX_shape: tuple[int, int],
+) -> ad.AnnData:
+    """
+    Produce a table with ROIS placed on a rectangular grid.
+
+    The main goal of this ROI grid is to allow processing of smaller subset of
+    the whole array.
+
+    In a specific case (that is, if the image array was obtained by stitching
+    together a set of FOVs placed on a regular grid), the ROIs correspond to
+    the original FOVs.
+
+    TODO: make this flexible with respect to the presence/absence of Z.
+
+    Args:
+        array_shape: ZYX shape of the image array.
+        pixels_ZYX: ZYX pixel sizes in micrometers.
+        grid_YX_shape:
+
+    Returns:
+        An `AnnData` table with a single ROI.
+    """
+    shape_z, shape_y, shape_x = array_shape[-3:]
+    grid_size_y, grid_size_x = grid_YX_shape[:]
+    X = []
+    obs_names = []
+    counter = 0
+    start_z = 0
+    len_z = shape_z
+
+    # Find minimal len_y that covers [0,shape_y] with grid_size_y intervals
+    len_y = math.ceil(shape_y / grid_size_y)
+    len_x = math.ceil(shape_x / grid_size_x)
+    for ind_y in range(grid_size_y):
+        start_y = ind_y * len_y
+        tmp_len_y = min(shape_y, start_y + len_y) - start_y
+        for ind_x in range(grid_size_x):
+            start_x = ind_x * len_x
+            tmp_len_x = min(shape_x, start_x + len_x) - start_x
+            X.append(
+                [
+                    start_x * pixels_ZYX[2],
+                    start_y * pixels_ZYX[1],
+                    start_z * pixels_ZYX[0],
+                    tmp_len_x * pixels_ZYX[2],
+                    tmp_len_y * pixels_ZYX[1],
+                    len_z * pixels_ZYX[0],
+                    start_x * pixels_ZYX[2],
+                    start_y * pixels_ZYX[1],
+                ]
+            )
+            counter += 1
+            obs_names.append(f"ROI_{counter}")
+    ROI_table = ad.AnnData(X=np.array(X, dtype=np.float32))
+    ROI_table.obs_names = obs_names
+    ROI_table.var_names = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+    ]
+    return ROI_table