feat: ignore unknown fields on internal output port definition instead of raising error

Author: elliotzh

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/_schema/component.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from marshmallow import fields, post_dump
+from marshmallow import fields, post_dump, EXCLUDE
 
 from azure.ai.ml._schema import NestedField, StringTransformedEnum, UnionField
 from azure.ai.ml._schema.component.component import ComponentSchema
@@ -66,8 +66,8 @@ class InternalBaseComponentSchema(ComponentSchema):
         keys=fields.Str(),
         values=UnionField(
             [
-                NestedField(InternalPrimitiveOutputSchema),
-                NestedField(InternalOutputPortSchema),
+                NestedField(InternalPrimitiveOutputSchema, unknown=EXCLUDE),
+                NestedField(InternalOutputPortSchema, unknown=EXCLUDE),
             ]
         ),
     )

sdk/ml/azure-ai-ml/tests/internal/_utils.py

@@ -11,7 +11,7 @@
 PARAMETERS_TO_TEST = [
     # which of them are available for other components?
     (
-        "tests/test_configs/internal/ls_command_component.yaml",
+        "tests/test_configs/internal/command-component-ls/ls_command_component.yaml",
         {},
         {
             "compute": "cpu-cluster",  # runsettings.target

sdk/ml/azure-ai-ml/tests/internal/e2etests/test_component.py

@@ -87,7 +87,10 @@ def test_component_load(
         component_resource = create_component(client, component_name, path=yaml_path)
         loaded_dict = load_registered_component(client, component_name, component_resource.version, omit_fields)
 
-        json_path = yaml_path.rsplit(".", 1)[0] + ".loaded_from_rest.json"
+        base_dir = "./tests/test_configs/internal"
+        json_path = (yaml_path.rsplit(".", 1)[0] + ".json")
+        json_path = os.path.join(base_dir, "loaded_from_rest", os.path.relpath(json_path, base_dir))
+        os.makedirs(os.path.dirname(json_path), exist_ok=True)
         if not os.path.isfile(json_path):
             with open(json_path, "w") as f:
                 json.dump(loaded_dict, f, indent=2)

sdk/ml/azure-ai-ml/tests/internal/e2etests/test_pipeline_job.py

@@ -105,7 +105,7 @@ def pipeline_func():
         "yaml_path,inputs,runsettings_dict,pipeline_runsettings_dict",
         PARAMETERS_TO_TEST,
     )
-    def test_anonymous_internal_component_in_pipeline(
+    def test_pipeline_anonymous(
         self, client: MLClient, yaml_path, inputs, runsettings_dict, pipeline_runsettings_dict
     ):
         # curated env with name & version

sdk/ml/azure-ai-ml/tests/internal/unittests/test_component.py

@@ -31,7 +31,7 @@ def test_load_v2_component(self):
 
     def test_specific_error_message_on_load_from_dict(self):
         os.environ[AZUREML_INTERNAL_COMPONENTS_ENV_VAR] = "false"
-        yaml_path = "./tests/test_configs/internal/helloworld_component_command.yml"
+        yaml_path = "./tests/test_configs/internal/helloworld/helloworld_component_command.yml"
         with pytest.raises(
             ValidationException,
             match="Internal components is a private feature in v2, " "please set environment variable",
@@ -439,6 +439,10 @@ def test_component_output_with_attrs(self) -> None:
         assert component
 
         expected_outputs = {
+            "path_with_optional": {
+                # unknown field optional will be ignored
+                "type": 'AnyDirectory',
+            },
             "primitive_is_control": {
                 "is_control": True,
                 "type": "boolean",
@@ -464,7 +468,7 @@ def test_loop_node_is_internal_components(self):
         from azure.ai.ml.constants._common import AZUREML_INTERNAL_COMPONENTS_ENV_VAR
         from azure.ai.ml.dsl._utils import environment_variable_overwrite
 
-        yaml_path = "./tests/test_configs/internal/helloworld_component_command.yml"
+        yaml_path = "./tests/test_configs/internal/helloworld/helloworld_component_command.yml"
         component_func = load_component(source=yaml_path)
         loop_node = LoopNode(body=component_func())
         loop_node.body._referenced_control_flow_node_instance_id = loop_node._instance_id

sdk/ml/azure-ai-ml/tests/internal/unittests/test_pipeline_job.py

@@ -164,7 +164,7 @@ def pipeline_func(pipeline_input):
 
     @pytest.mark.usefixtures("enable_pipeline_private_preview_features")
     def test_internal_component_output_as_pipeline_component_output(self):
-        yaml_path = "./tests/test_configs/internal/component_with_input_types/component_spec.yaml"
+        yaml_path = "./tests/test_configs/internal/component_with_input_outputs/component_spec.yaml"
         component_func = load_component(yaml_path, params_override=[{"inputs": {}}])
 
         @pipeline()
@@ -220,7 +220,7 @@ def pipeline_func():
         assert result._to_dict() == {"result": "Succeeded"}
 
     def test_gjd_internal_component_in_pipeline(self):
-        yaml_path = "./tests/test_configs/internal/ls_command_component.yaml"  # GJD is based on CommandComponent
+        yaml_path = "./tests/test_configs/internal/command-component-ls/ls_command_component.yaml"  # GJD is based on CommandComponent
         node_func: CommandComponent = load_component(yaml_path)
         node = node_func()
 
@@ -246,7 +246,7 @@ def test_gjd_internal_component_in_pipeline(self):
 
     def test_elastic_component_in_pipeline(self):
         yaml_path = (
-            "./tests/test_configs/internal/ls_command_component.yaml"  # itp & elastic are based on CommandComponent
+            "./tests/test_configs/internal/command-component-ls/ls_command_component.yaml"  # itp & elastic are based on CommandComponent
         )
         node_func: CommandComponent = load_component(yaml_path)
         node = node_func()
@@ -303,7 +303,7 @@ def test_elastic_component_in_pipeline(self):
 
     def test_singularity_component_in_pipeline(self):
         yaml_path = (
-            "./tests/test_configs/internal/ls_command_component.yaml"  # singularity is based on CommandComponent
+            "./tests/test_configs/internal/command-component-ls/ls_command_component.yaml"  # singularity is based on CommandComponent
         )
         node_func: CommandComponent = load_component(yaml_path)
         node = node_func()
@@ -366,7 +366,7 @@ def test_singularity_component_in_pipeline(self):
         }
 
     def test_load_pipeline_job_with_internal_components_as_node(self):
-        yaml_path = Path("./tests/test_configs/internal/helloworld_component_scope.yml")
+        yaml_path = Path("./tests/test_configs/internal/helloworld/helloworld_component_scope.yml")
         scope_internal_func = load_component(source=yaml_path)
         with open(yaml_path, encoding="utf-8") as yaml_file:
             yaml_dict = yaml.safe_load(yaml_file)
@@ -472,7 +472,7 @@ def pipeline_func():
         assert dsl_pipeline._to_dict() == regenerated_pipeline_job._to_dict()
 
     def test_components_input_output(self):
-        yaml_path = "./tests/test_configs/internal/component_with_input_types/component_spec.yaml"
+        yaml_path = "./tests/test_configs/internal/component_with_input_outputs/component_spec.yaml"
         component: InternalComponent = load_component(yaml_path)
 
         fake_input = Input(type=AssetTypes.MLTABLE, path="azureml:scope_tsv:1")
@@ -529,7 +529,7 @@ def pipeline_func():
         assert rest_obj.properties.jobs["node"]["inputs"] == expected_inputs
 
     def test_data_binding_on_node_runsettings(self):
-        test_path = "./tests/test_configs/internal/helloworld_component_command.yml"
+        test_path = "./tests/test_configs/internal/helloworld/helloworld_component_command.yml"
         component: InternalComponent = load_component(test_path)
 
         @pipeline()