feat: support setting path for node output directly (#26712)

* feat: support setting path for node output directly

* fix: fix ci

Author: elliotzh

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/pipeline/_io.py

@@ -155,6 +155,8 @@ def path(self) -> str:
 
     @path.setter
     def path(self, path):
+        # For un-configured input/output, we build a default data entry for them.
+        self._build_default_data()
         if hasattr(self._data, "path"):
             self._data.path = path
         else:
@@ -361,7 +363,9 @@ def is_control(self) -> str:
     def _build_default_data(self):
         """Build default data when output not configured."""
         if self._data is None:
-            self._data = Output()
+            # _meta will be None when node._component is not a Component object
+            # so we just leave the type inference work to backend
+            self._data = Output(type=None)
 
     def _build_data(self, data, key=None):
         """Build output data according to assigned input, eg: node.outputs.key = data"""
@@ -593,15 +597,13 @@ def _validate_inputs(cls, inputs):
 
     def __getattr__(self, name: K) -> V:
         if name not in self:
-            # pylint: disable=unnecessary-comprehension
-            raise UnexpectedAttributeError(keyword=name, keywords=[key for key in self])
+            raise UnexpectedAttributeError(keyword=name, keywords=list(self))
         return super().__getitem__(name)
 
     def __getitem__(self, item: K) -> V:
         # We raise this exception instead of KeyError
         if item not in self:
-            # pylint: disable=unnecessary-comprehension
-            raise UnexpectedKeywordError(func_name="ParameterGroup", keyword=item, keywords=[key for key in self])
+            raise UnexpectedKeywordError(func_name="ParameterGroup", keyword=item, keywords=list(self))
         return super().__getitem__(item)
 
     # For Jupyter Notebook auto-completion
@@ -654,6 +656,13 @@ def __init__(self, outputs: dict, **kwargs):
     def __getattr__(self, item) -> NodeOutput:
         return self.__getitem__(item)
 
+    def __getitem__(self, item) -> NodeOutput:
+        if item not in self:
+            # We raise this exception instead of KeyError as OutputsAttrDict doesn't support add new item after
+            # __init__.
+            raise UnexpectedAttributeError(keyword=item, keywords=list(self))
+        return super().__getitem__(item)
+
     def __setattr__(self, key: str, value: Union[Data, Output]):
         if isinstance(value, Output):
             mode = value.mode

sdk/ml/azure-ai-ml/tests/dsl/unittests/test_component_func.py

@@ -1,18 +1,16 @@
 from pathlib import Path
 from typing import Callable, Union
 
-import marshmallow
 import pytest
 from marshmallow import ValidationError
 
 from azure.ai.ml import PyTorchDistribution, load_component
-from azure.ai.ml.entities import Component as ComponentEntity
 from azure.ai.ml.entities import Data, JobResourceConfiguration
 from azure.ai.ml.entities._builders import Command
 from azure.ai.ml.entities._inputs_outputs import Input, Output
 from azure.ai.ml.entities._job.pipeline._io import PipelineInput, PipelineOutput
 from azure.ai.ml.entities._job.pipeline._load_component import _generate_component_function
-from azure.ai.ml.exceptions import UnexpectedKeywordError, ValidationException
+from azure.ai.ml.exceptions import UnexpectedKeywordError, ValidationException, UnexpectedAttributeError
 
 from .._util import _DSL_TIMEOUT_SECOND
 
@@ -156,7 +154,22 @@ def test_component_outputs(self):
 
         # configure mode and default Output is built
         component.outputs.component_out_path.mode = "upload"
-        assert component._build_outputs() == {"component_out_path": Output(mode="upload")}
+        assert component._build_outputs() == {"component_out_path": Output(type=None, mode="upload")}
+
+        test_output_path = "azureml://datastores/workspaceblobstore/paths/azureml/ps_copy_component/outputs/output_dir"
+        component: Command = component_func()
+
+        # configure path and default Output is built
+        component.outputs.component_out_path.path = test_output_path
+        assert component._build_outputs() == {"component_out_path": Output(type=None, path=test_output_path)}
+
+        # non-existent output
+        with pytest.raises(
+                UnexpectedAttributeError,
+                match="Got an unexpected attribute 'component_out_path_non', "
+                      "valid attributes: 'component_out_path'."
+        ):
+            component.outputs["component_out_path_non"].path = test_output_path
 
         # configure data
         component: Command = component_func()

sdk/ml/azure-ai-ml/tests/dsl/unittests/test_dsl_pipeline.py

@@ -1,4 +1,3 @@
-import logging
 import os
 from functools import partial
 from io import StringIO
@@ -585,7 +584,7 @@ def pipeline(
             }
 
         job_yaml = "./tests/test_configs/pipeline_jobs/helloworld_pipeline_job_data_options.yml"
-        pipeline_job = load_job(source=job_yaml)
+        pipeline_job: PipelineJob = load_job(source=job_yaml)
 
         pipeline = pipeline(**{key: val for key, val in pipeline_job._build_inputs().items()})
         pipeline.inputs.job_in_data_by_store_path_and_mount.mode = "ro_mount"
@@ -605,6 +604,8 @@ def pipeline(
         actual_outputs = pipeline._build_outputs()
         for k, v in actual_outputs.items():
             v.mode = v.mode.lower()
+            # outputs defined in yaml are all uri_folder, while its default value in dsl is None
+            v.type = "uri_folder"
         assert pipeline_job._build_outputs() == actual_outputs
 
         component_job = next(iter(pipeline_job.jobs.values()))._to_rest_object()

sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py

@@ -527,3 +527,16 @@ def pipeline_func():
             if key.startswith("data_"):
                 expected_inputs[key] = {"job_input_type": "mltable", "uri": "azureml:scope_tsv:1"}
         assert rest_obj.properties.jobs["node"]["inputs"] == expected_inputs
+
+    def test_pipeline_with_setting_node_output_directly(self) -> None:
+        component_dir = Path(__file__).parent.parent.parent / "test_configs" / "internal" / "command-component"
+        copy_func = load_component(component_dir / "command-linux/copy/component.yaml")
+
+        copy_file = copy_func(
+            input_dir=None,
+            file_names=None,
+        )
+
+        copy_file.outputs.output_dir.path = "path_on_datastore"
+        assert copy_file.outputs.output_dir.path == "path_on_datastore"
+        assert copy_file.outputs.output_dir.type == "path"

sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py

@@ -1454,3 +1454,17 @@ def test_comment_in_pipeline(self) -> None:
         rest_pipeline_dict = pipeline_job._to_rest_object().as_dict()["properties"]
         assert pipeline_dict["jobs"]["hello_world_component"]["comment"] == "arbitrary string"
         assert rest_pipeline_dict["jobs"]["hello_world_component"]["comment"] == "arbitrary string"
+
+    def test_pipeline_node_default_output(self):
+        test_path = "./tests/test_configs/pipeline_jobs/helloworld_pipeline_job_with_component_output.yml"
+        pipeline: PipelineJob = load_job(source=test_path)
+
+        # pipeline level output
+        pipeline_output = pipeline.outputs["job_out_path_2"]
+        assert pipeline_output.mode == "upload"
+
+        # other node level output tests can be found in
+        # dsl/unittests/test_component_func.py::TestComponentFunc::test_component_outputs
+        # data-binding-expression
+        with pytest.raises(ValidationException, match="<class '.*'> does not support setting path."):
+            pipeline.jobs["merge_component_outputs"].outputs["component_out_path_1"].path = "xxx"