fix: combo fix for internal components I

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/_schema/component.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from marshmallow import fields, post_dump, EXCLUDE
+from marshmallow import fields, post_dump, INCLUDE, EXCLUDE
 
 from azure.ai.ml._schema import NestedField, StringTransformedEnum, UnionField
 from azure.ai.ml._schema.component.component import ComponentSchema
@@ -43,6 +43,8 @@ def all_values(cls):
 
 
 class InternalBaseComponentSchema(ComponentSchema):
+    class Meta:
+        unknown = INCLUDE
     # override name as 1p components allow . in name, which is not allowed in v2 components
     name = fields.Str()
 

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/_schema/node.py

@@ -12,6 +12,8 @@
 
 
 class InternalBaseNodeSchema(BaseNodeSchema):
+    class Meta:
+        unknown = INCLUDE
     component = UnionField(
         [
             # for registry type assets

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/_additional_includes.py

@@ -16,19 +16,34 @@
 
 class _AdditionalIncludes:
     def __init__(self, code_path: Union[None, str], yaml_path: str):
-        self._yaml_path = Path(yaml_path)
-        self._yaml_name = self._yaml_path.name
-        self._code_path = self._yaml_path.parent
-        if code_path is not None:
-            self._code_path = (self._code_path / code_path).resolve()
+        self.__yaml_path = Path(yaml_path)
+        self.__code_path = code_path
+
         self._tmp_code_path = None
-        self._additional_includes_file_path = self._yaml_path.with_suffix(f".{ADDITIONAL_INCLUDES_SUFFIX}")
         self._includes = None
         if self._additional_includes_file_path.is_file():
             with open(self._additional_includes_file_path, "r") as f:
                 lines = f.readlines()
                 self._includes = [line.strip() for line in lines if len(line.strip()) > 0]
 
+    @property
+    def _yaml_path(self) -> Path:
+        return self.__yaml_path
+
+    @property
+    def _code_path(self) -> Path:
+        if self.__code_path is not None:
+            return (self._yaml_path.parent / self.__code_path).resolve()
+        return self._yaml_path.parent
+
+    @property
+    def _yaml_name(self) -> str:
+        return self._yaml_path.name
+
+    @property
+    def _additional_includes_file_path(self) -> Path:
+        return self._yaml_path.with_suffix(f".{ADDITIONAL_INCLUDES_SUFFIX}")
+
     @property
     def code(self) -> Path:
         return self._tmp_code_path if self._tmp_code_path else self._code_path

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/component.py

@@ -5,6 +5,7 @@
 # disable redefined-builtin to use id/type as argument name
 from contextlib import contextmanager
 from typing import Dict, Union
+import os
 
 from marshmallow import INCLUDE, Schema
 
@@ -176,6 +177,21 @@ def _additional_includes(self):
     def _create_schema_for_validation(cls, context) -> Union[PathAwareSchema, Schema]:
         return InternalBaseComponentSchema(context=context)
 
+    def _validate(self, raise_error=False) -> MutableValidationResult:
+        if self._additional_includes is not None and self._additional_includes._validate().passed:
+            # update source path in case dependency file is in additional_includes
+            with self._resolve_local_code() as tmp_base_path:
+                origin_base_path, origin_source_path = self._base_path, self._source_path
+
+                try:
+                    self._base_path, self._source_path = \
+                        tmp_base_path, tmp_base_path / os.path.basename(self._source_path)
+                    return super()._validate(raise_error=raise_error)
+                finally:
+                    self._base_path, self._source_path = origin_base_path, origin_source_path
+
+        return super()._validate(raise_error=raise_error)
+
     def _customized_validate(self) -> MutableValidationResult:
         validation_result = super(InternalComponent, self)._customized_validate()
         if isinstance(self.environment, InternalEnvironment):
@@ -228,14 +244,3 @@ def _resolve_local_code(self):
 
     def __call__(self, *args, **kwargs) -> InternalBaseNode:  # pylint: disable=useless-super-delegation
         return super(InternalComponent, self).__call__(*args, **kwargs)
-
-    def _schema_validate(self) -> MutableValidationResult:
-        """Validate the resource with the schema.
-
-        return type: ValidationResult
-        """
-        result = super(InternalComponent, self)._schema_validate()
-        # skip unknown field warnings for internal components
-        # TODO: move this logic into base class
-        result._warnings = list(filter(lambda x: x.message != "Unknown field.", result._warnings))
-        return result

sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/node.py

@@ -16,7 +16,6 @@
 from azure.ai.ml.entities._job.pipeline._io import NodeInput, NodeOutput, PipelineInput
 from azure.ai.ml.entities._util import convert_ordered_dict_to_dict
 
-from ...entities._validation import MutableValidationResult
 from .._schema.component import NodeType
 from ._input_outputs import InternalInput
 
@@ -95,17 +94,6 @@ def _to_job(self) -> Job:
     def _load_from_dict(cls, data: Dict, context: Dict, additional_message: str, **kwargs) -> "Job":
         raise RuntimeError("Internal components doesn't support load from dict")
 
-    def _schema_validate(self) -> MutableValidationResult:
-        """Validate the resource with the schema.
-
-        return type: ValidationResult
-        """
-        result = super(InternalBaseNode, self)._schema_validate()
-        # skip unknown field warnings for internal components
-        # TODO: move this logic into base class?
-        result._warnings = list(filter(lambda x: x.message != "Unknown field.", result._warnings))
-        return result
-
     @classmethod
     def _create_schema_for_validation(cls, context) -> Union[PathAwareSchema, Schema]:
         from .._schema.node import InternalBaseNodeSchema

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/core/schema_meta.py

@@ -44,8 +44,10 @@ def __new__(cls, name, bases, dct):
         if meta is None:
             dct["Meta"] = PatchedMeta
         else:
-            dct["Meta"].unknown = RAISE
-            dct["Meta"].ordered = True
+            if not hasattr(meta, "unknown"):
+                dct["Meta"].unknown = RAISE
+            if not hasattr(meta, "ordered"):
+                dct["Meta"].ordered = True
 
         bases = bases + (PatchedBaseSchema,)
         klass = super().__new__(cls, name, bases, dct)

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_validation.py

@@ -420,7 +420,7 @@ def _validate(self, raise_error=False) -> MutableValidationResult:
         :type raise_error: bool
         return type: ValidationResult
         """
-        result = self._schema_validate()
+        result = self.__schema_validate()
         result.merge_with(self._customized_validate())
         return result.try_raise(
             raise_error=raise_error, error_target=self._get_validation_error_target()
@@ -443,7 +443,7 @@ def _get_skip_fields_in_schema_validation(
         """
         return []
 
-    def _schema_validate(self) -> MutableValidationResult:
+    def __schema_validate(self) -> MutableValidationResult:
         """Validate the resource with the schema.
 
         return type: ValidationResult

sdk/ml/azure-ai-ml/tests/internal/unittests/test_component.py

@@ -319,7 +319,7 @@ def test_additional_includes(self) -> None:
             "./tests/test_configs/internal/component_with_additional_includes/helloworld_additional_includes.yml"
         )
         component: InternalComponent = load_component(source=yaml_path)
-        assert component._validate().passed, component._validate()._to_dict()
+        assert component._validate().passed, repr(component._validate())
         # resolve
         with component._resolve_local_code() as code_path:
             assert code_path.is_dir()
@@ -340,7 +340,7 @@ def test_additional_includes(self) -> None:
     def test_additional_includes_with_code_specified(self, yaml_path: str, has_additional_includes: bool) -> None:
         yaml_path = os.path.join("./tests/test_configs/internal/component_with_additional_includes/", yaml_path)
         component: InternalComponent = load_component(source=yaml_path)
-        assert component._validate().passed, component._validate()._to_dict()
+        assert component._validate().passed, repr(component._validate())
         # resolve
         with component._resolve_local_code() as code_path:
             assert code_path.is_dir()
@@ -355,6 +355,12 @@ def test_additional_includes_with_code_specified(self, yaml_path: str, has_addit
                 specified_code_path = Path(yaml_path).parent / yaml_dict.get("code", "./")
                 assert code_path.resolve() == specified_code_path.resolve()
 
+    def test_docker_file_in_additional_includes(self):
+        yaml_path = "./tests/test_configs/internal/component_with_docker_file_" \
+                    "in_additional_includes/helloworld_additional_includes.yml"
+        component: InternalComponent = load_component(source=yaml_path)
+        assert component._validate().passed, repr(component._validate())
+
     @pytest.mark.parametrize(
         "yaml_path,expected_error_msg_prefix",
         [

sdk/ml/azure-ai-ml/tests/test_configs/internal/additional_includes/docker/DockerFile

@@ -0,0 +1,32 @@
+FROM ubuntu:16.04
+
+ARG CONDA_VERSION=4.7.12
+ARG PYTHON_VERSION=3.7
+ARG AZUREML_SDK_VERSION=1.33.0
+ARG INFERENCE_SCHEMA_VERSION=1.3.0
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PATH /opt/miniconda/bin:$PATH
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y wget bzip2 && \
+    apt-get install -y fuse && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd --create-home dockeruser
+WORKDIR /home/dockeruser
+USER dockeruser
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p ~/miniconda && \
+    rm ~/miniconda.sh && \
+    ~/miniconda/bin/conda clean -tipsy
+ENV PATH="/home/dockeruser/miniconda/bin/:${PATH}"
+
+RUN conda install -y conda=${CONDA_VERSION} python=${PYTHON_VERSION} && \
+    pip install azureml-defaults==${AZUREML_SDK_VERSION} inference-schema==${INFERENCE_SCHEMA_VERSION} &&\
+    conda clean -aqy && \
+    rm -rf ~/miniconda/pkgs && \
+    find ~/miniconda/ -type d -name __pycache__ -prune -exec rm -rf {} \;

sdk/ml/azure-ai-ml/tests/test_configs/internal/component_with_docker_file_in_additional_includes/helloworld_additional_includes.additional_includes

@@ -0,0 +1 @@
+../additional_includes/docker/DockerFile

sdk/ml/azure-ai-ml/tests/test_configs/internal/component_with_docker_file_in_additional_includes/helloworld_additional_includes.yml

@@ -0,0 +1,33 @@
+$schema: https://componentsdk.azureedge.net/jsonschema/CommandComponent.json
+name: microsoft.com.azureml.samples.train
+version: 0.0.4
+display_name: Train
+type: CommandComponent
+description: A dummy training module
+tags: {category: Component Tutorial, contact: [email protected]}
+inputs:
+  training_data:
+    type: path
+    description: Training data organized in the torchvision format/structure
+    optional: false
+  max_epochs:
+    type: integer
+    description: Maximum number of epochs for the training
+    optional: false
+  learning_rate:
+    type: float
+    description: Learning rate, default is 0.01
+    default: 0.01
+    optional: false
+outputs:
+  model_output:
+    type: path
+    description: The output model
+command: >-
+  python train.py --training_data {inputs.training_data} --max_epochs {inputs.max_epochs}
+  --learning_rate {inputs.learning_rate} --model_output {outputs.model_output}
+environment:
+  docker:
+    build:
+      dockerfile: file:DockerFile
+  os: Linux