From 086e24b6da389f999a690d89e3cbf1d421ee1b0f Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Tue, 14 May 2024 15:05:08 +0530 Subject: [PATCH 01/28] fixing sweep job service response parsing issue --- sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py index d8a3fa797fb8..abb4529ae03c 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py @@ -60,6 +60,9 @@ def _from_rest_object( if obj is None: return None + if isinstance(obj, DistributionConfiguration): + return obj + data = obj if isinstance(obj, RestDistributionConfiguration): data = obj.as_dict() From 2d60c0945de7dc45c70cfd7e4de0e756072292b5 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Tue, 14 May 2024 17:51:08 +0530 Subject: [PATCH 02/28] fixing sweep job service response parsing issue --- .../azure-ai-ml/azure/ai/ml/entities/_job/distribution.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py index abb4529ae03c..ec7277c647eb 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/distribution.py @@ -60,11 +60,9 @@ def _from_rest_object( if obj is None: return None - if isinstance(obj, DistributionConfiguration): - return obj - - data = obj - if isinstance(obj, RestDistributionConfiguration): + if isinstance(obj, dict): + data = obj + else: data = obj.as_dict() type_str = data.pop("distribution_type", None) or data.pop("type", None) From 11bc1563a523bce66b994cb566a2dd8b1083366a Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Fri, 17 May 2024 16:33:13 +0530 Subject: [PATCH 03/28] raising a work around for cmd job and spark job name lowering case --- sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py index 9550fcff30ae..5a3c0b2ee8e2 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py @@ -306,9 +306,13 @@ def _from_rest_object( # pylint: disable=too-many-return-statements return ImportJob._load_from_rest(obj) res_command: Job = Command._load_from_rest_job(obj) + if hasattr(obj, "name"): + res_command._name = obj.name return res_command if obj.properties.job_type == RestJobType.SPARK: res_spark: Job = Spark._load_from_rest_job(obj) + if hasattr(obj, "name"): + res_spark._name = obj.name return res_spark if obj.properties.job_type == RestJobType.SWEEP: return SweepJob._load_from_rest(obj) From 0cab5911fbb6b39c726627dedae41c61e2948735 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Fri, 17 May 2024 18:06:23 +0530 Subject: [PATCH 04/28] fixing black issue --- sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py index 5a3c0b2ee8e2..6d4dc5f8c7b0 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/job.py @@ -307,12 +307,12 @@ def _from_rest_object( # pylint: disable=too-many-return-statements res_command: Job = Command._load_from_rest_job(obj) if hasattr(obj, "name"): - res_command._name = obj.name + res_command._name = obj.name # type: ignore[attr-defined] return res_command if obj.properties.job_type == RestJobType.SPARK: res_spark: Job = Spark._load_from_rest_job(obj) if hasattr(obj, "name"): - res_spark._name = obj.name + res_spark._name = obj.name # type: ignore[attr-defined] return res_spark if obj.properties.job_type == RestJobType.SWEEP: return SweepJob._load_from_rest(obj) From e6a61779c8091ef4c1bf455d6661eaa6a613a318 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Mon, 20 May 2024 17:40:06 +0530 Subject: [PATCH 05/28] fixing serialization issue in sweep job --- sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py index 6a0d9857e271..970391b0263a 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py @@ -323,7 +323,7 @@ def _load_from_rest(cls, obj: JobBase) -> "SweepJob": search_space=_search_space, # type: ignore[arg-type] limits=SweepJobLimits._from_rest_object(properties.limits), early_termination=early_termination, - objective=properties.objective, + objective=Objective._from_rest_object(properties.objective) if properties.objective else None, inputs=from_rest_inputs_to_dataset_literal(properties.inputs), outputs=from_rest_data_outputs(properties.outputs), identity=( From 54f97dbe71776fbcc507f6803e7a992e6401e078 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Mon, 20 May 2024 22:38:38 +0530 Subject: [PATCH 06/28] include black fix --- .../azure/ai/ml/operations/_job_operations.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py index ae49808ff6c2..b9cd93163f28 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py @@ -1473,11 +1473,16 @@ def _resolve_arm_id_for_sweep_job(self, job: SweepJob, resolver: _AssetResolver) Code(base_path=job._base_path, path=job.trial.code), azureml_type=AzureMLResourceType.CODE, ) - if job.trial is not None: + if ( + job.trial is not None + and job.trial.environment is not None + and not is_ARM_id_for_resource(job.trial.environment, AzureMLResourceType.ENVIRONMENT) + ): job.trial.environment = resolver( # type: ignore[assignment] job.trial.environment, azureml_type=AzureMLResourceType.ENVIRONMENT ) - job.compute = self._resolve_compute_id(resolver, job.compute) + if job.compute is not None and not is_ARM_id_for_resource(job.compute, AzureMLResourceType.COMPUTE): + job.compute = self._resolve_compute_id(resolver, job.compute) return job def _resolve_arm_id_for_automl_job( From b6647ba9bf286fee32a9c4e9bd9ae4a99409d5b4 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Mon, 20 May 2024 23:49:12 +0530 Subject: [PATCH 07/28] found and fix the issue in compute validation --- .../azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py | 4 ++++ sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py index 970391b0263a..fea84e28cc1a 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/sweep/sweep_job.py @@ -268,6 +268,10 @@ def _to_rest_object(self) -> JobBase: self.resources._to_rest_object() if self.resources and not isinstance(self.resources, dict) else None ), ) + + if not sweep_job.resources and sweep_job.trial.resources: + sweep_job.resources = sweep_job.trial.resources + sweep_job_resource = JobBase(properties=sweep_job) sweep_job_resource.name = self.name return sweep_job_resource diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py index b9cd93163f28..97d1fe310d27 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py @@ -1481,8 +1481,7 @@ def _resolve_arm_id_for_sweep_job(self, job: SweepJob, resolver: _AssetResolver) job.trial.environment = resolver( # type: ignore[assignment] job.trial.environment, azureml_type=AzureMLResourceType.ENVIRONMENT ) - if job.compute is not None and not is_ARM_id_for_resource(job.compute, AzureMLResourceType.COMPUTE): - job.compute = self._resolve_compute_id(resolver, job.compute) + job.compute = self._resolve_compute_id(resolver, job.compute) return job def _resolve_arm_id_for_automl_job( From 68cffc1e5a6232eea2f59110372943fa08ac3668 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Thu, 23 May 2024 14:59:27 +0530 Subject: [PATCH 08/28] fixing a corrupt wokspace arm template --- .../ai/ml/_arm_deployments/arm_templates/workspace_base.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json b/sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json index ef27f7e3551d..28b38b741645 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_arm_deployments/arm_templates/workspace_base.json @@ -795,9 +795,8 @@ "description": "[parameters('description')]", "storageAccount": "[variables('storageAccount')]", "keyVault": "[variables('keyVault')]", - "applicationInsights": "[variables('applicationInsights')]", "containerRegistry": "[if(not(equals(parameters('containerRegistryOption'), 'none')), variables('containerRegistry'), json('null'))]", - "applicationInsights": "[if(not(equals(parameters('applicationInsightsOption'), 'none')), variables('containerRegistry'), json('null'))]", + "applicationInsights": "[if(not(equals(parameters('applicationInsightsOption'), 'none')), variables('applicationInsights'), json('null'))]", "hbiWorkspace": "[parameters('confidential_data')]", "imageBuildCompute": "[parameters('imageBuildCompute')]", "publicNetworkAccess": "[parameters('publicNetworkAccess')]", From ff1caf9dc9cbda2851fd835ef4c6f21ccb753497 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Fri, 31 May 2024 11:22:45 +0530 Subject: [PATCH 09/28] fixing the failed test case --- .../azure-ai-ml/tests/sweep_job/e2etests/test_sweep_job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/tests/sweep_job/e2etests/test_sweep_job.py b/sdk/ml/azure-ai-ml/tests/sweep_job/e2etests/test_sweep_job.py index 613fea739c5c..fb92613650ac 100644 --- a/sdk/ml/azure-ai-ml/tests/sweep_job/e2etests/test_sweep_job.py +++ b/sdk/ml/azure-ai-ml/tests/sweep_job/e2etests/test_sweep_job.py @@ -209,7 +209,10 @@ def test_sweep_job_builder(self, randstr: Callable[[str], str], client: MLClient result = client.create_or_update(sweep_node) assert result.description == "new-description" - assert result.trial.environment == "AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:33" + assert ( + result.trial.environment + == "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/environments/AzureML-sklearn-1.0-ubuntu20.04-py38-cpu/versions/33" + ) assert result.display_name == "new_builder_command_job" assert result.compute == "testCompute" assert result.experiment_name == "mfe-test1-dataset" From e7fb030b8ed3465027fbefdf3e9ff7eedf249957 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Mon, 1 Jul 2024 13:44:38 +0530 Subject: [PATCH 10/28] updating the date for 1.18.0 release --- sdk/ml/azure-ai-ml/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 159c72521072..aff12f6df2fe 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.18.0 (unreleased) +## 1.18.0 (2024-07-09) ### Features Added From d167e9065b5123c03b74c0ba1e171257dfdffce5 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Mon, 1 Jul 2024 15:19:23 +0530 Subject: [PATCH 11/28] removing empty section --- sdk/ml/azure-ai-ml/CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index aff12f6df2fe..12bdfcacb109 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -10,8 +10,6 @@ Expose `public_ip_address` in `AmlComputeNodeInfo`, to get the public ip address - InputTypes exported in constants module - WorkspaceConnection tags are now listed as deprecated, and the erroneously-deprecated metadata field has been un-deprecated and added as a initialization field. These two fields still point to the same underlying object property, and actual API usage of this value is unchanged. -### Breaking Changes - ### Other Changes - WorkspaceConnections are officially GA'd and no longer experimental. But its much newer subclasses remain experimental. From a6d644ac4005f4e1c8e3a10885483e82df3b01a7 Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Tue, 2 Jul 2024 10:56:08 +0530 Subject: [PATCH 12/28] adding in progress version details --- sdk/ml/azure-ai-ml/CHANGELOG.md | 2 ++ sdk/ml/azure-ai-ml/azure/ai/ml/_version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 12bdfcacb109..25b7e117ed81 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -1,5 +1,7 @@ # Release History +## 1.19.0 (unreleased) + ## 1.18.0 (2024-07-09) ### Features Added diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_version.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_version.py index a024f4054106..f711dd6e21c0 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_version.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_version.py @@ -2,4 +2,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -VERSION = "1.18.0" +VERSION = "1.19.0" From 248fb11bab1d44db7ddb8702cab0c7ea857b382c Mon Sep 17 00:00:00 2001 From: Amit Chauhan Date: Wed, 3 Jul 2024 09:11:29 +0530 Subject: [PATCH 13/28] adding IDC code reviewers --- .github/CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d12b8710d02d..ddd0cdac1f48 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -42,11 +42,11 @@ # Service team #### -# ServiceOwners: @luigiw @needuv @paulshealy1 @singankit +# ServiceOwners: @luigiw @needuv @achauhan-scc @kingernupur @paulshealy1 @singankit # ServiceLabel: %AI # PRLabel: %AI -/sdk/ai/ @luigiw @needuv @paulshealy1 @singankit +/sdk/ai/ @luigiw @needuv @achauhan-scc @kingernupur @paulshealy1 @singankit # ServiceLabel: %AgriFood # PRLabel: %AgriFood @@ -251,7 +251,7 @@ # ServiceOwners: @azureml-github @Azure/azure-ml-sdk # PRLabel: %Machine Learning -/sdk/ml/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @ninghu @YusakuNo1 +/sdk/ml/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @achauhan-scc @kingernupur @ninghu @YusakuNo1 # PRLabel: %ML-Data /sdk/ml/azure-ai-ml/azure/ai/ml/_schema/_data/ @sogansky @code-vicar @pritamDas9191 @@ -333,8 +333,8 @@ /sdk/ml/azure-ai-ml/azure/ai/ml/operations/_workspace_operations.py @seanyao1 @debuggerXi # PRLabel: %ML-WorkspaceConnections -/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/connections/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @ninghu @YusakuNo1 -/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_workspace/connections/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @ninghu @YusakuNo1 +/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/workspace/connections/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @achauhan-scc @kingernupur @ninghu @YusakuNo1 +/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_workspace/connections/ @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @achauhan-scc @kingernupur @ninghu @YusakuNo1 # PRLabel: %ML-ImportJob /sdk/ml/azure-ai-ml/azure/ai/ml/_schema/job/import_job.py @xiongrenyi From 4fc5d3cb5df1a4525e4b3f6594af5e433534b800 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:07:22 +0530 Subject: [PATCH 14/28] correcting the directory path --- sdk/ml/azure-ai-ml/samples/ml_samples_featurestore.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/ml/azure-ai-ml/samples/ml_samples_featurestore.py b/sdk/ml/azure-ai-ml/samples/ml_samples_featurestore.py index 189c0672c852..340bd3cc7b0f 100644 --- a/sdk/ml/azure-ai-ml/samples/ml_samples_featurestore.py +++ b/sdk/ml/azure-ai-ml/samples/ml_samples_featurestore.py @@ -97,9 +97,7 @@ def feature_store(self): description="7-day and 3-day rolling aggregation of transactions featureset", entities=["azureml:account:1"], stage="Development", - specification=FeatureSetSpecification( - path="../tests/test_configs/feature_set/code_sample/FeatureSetSpec.yaml" - ), + specification=FeatureSetSpecification(path="../azure-ai-ml/tests/test_configs/feature_set/code_sample/"), tags={"data_type": "nonPII"}, ) From 28a84b45ac716f5325994f9581e09b71e78f12e1 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:35:58 +0530 Subject: [PATCH 15/28] lint fix --- sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py | 3 ++- .../ai/ml/operations/_compute_operations.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py index 70195e7d14f2..96236855e178 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_ml_client.py @@ -78,7 +78,6 @@ BatchEndpointOperations, ComponentOperations, ComputeOperations, - WorkspaceConnectionsOperations, DataOperations, DatastoreOperations, EnvironmentOperations, @@ -91,6 +90,7 @@ OnlineEndpointOperations, RegistryOperations, ServerlessEndpointOperations, + WorkspaceConnectionsOperations, WorkspaceOperations, ) from azure.ai.ml.operations._code_operations import CodeOperations @@ -510,6 +510,7 @@ def __init__( self._operation_scope, self._operation_config, self._service_client_08_2023_preview, + self._service_client_04_2024_preview, **app_insights_handler_kwargs, # type: ignore[arg-type] ) self._operation_container.add(AzureMLResourceType.COMPUTE, self._compute) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py index 5d894ed2b014..57b53fa10132 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py @@ -7,8 +7,11 @@ from typing import Any, Dict, Iterable, Optional, cast from azure.ai.ml._restclient.v2023_08_01_preview import AzureMachineLearningWorkspaces as ServiceClient022023Preview +from azure.ai.ml._restclient.v2024_04_01_preview import AzureMachineLearningWorkspaces as ServiceClient042024Preview +from azure.ai.ml._restclient.v2024_04_01_preview.models import SsoSetting from azure.ai.ml._scope_dependent_operations import OperationConfig, OperationScope, _ScopeDependentOperations from azure.ai.ml._telemetry import ActivityType, monitor_with_activity +from azure.ai.ml._utils._experimental import experimental from azure.ai.ml._utils._logger_utils import OpsLogger from azure.ai.ml.constants._common import COMPUTE_UPDATE_ERROR from azure.ai.ml.constants._compute import ComputeType @@ -39,11 +42,13 @@ def __init__( operation_scope: OperationScope, operation_config: OperationConfig, service_client: ServiceClient022023Preview, + service_client_2024: ServiceClient042024Preview, **kwargs: Dict, ) -> None: super(ComputeOperations, self).__init__(operation_scope, operation_config) ops_logger.update_info(kwargs) self._operation = service_client.compute + self._operation2024 = service_client_2024.compute self._workspace_operations = service_client.workspaces self._vmsize_operations = service_client.virtual_machine_sizes self._usage_operations = service_client.usages @@ -416,6 +421,26 @@ def list_sizes(self, *, location: Optional[str] = None, compute_type: Optional[s ] return [VmSize._from_rest_object(item) for item in size_list.value] + @distributed_trace + @monitor_with_activity(ops_logger, "Compute.enablesso", ActivityType.PUBLICAPI) + @experimental + def enable_sso(self, *, name: str, enable_sso: Optional[bool] = True): + """enable sso for a compute instance. + + :keyword name: Name of the compute instance. + :paramtype name: str + :keyword enable_sso: enable sso bool flag + Default to True + :paramtype enable_sso: bool + """ + + self._operation2024.update_sso_settings( + self._operation_scope.resource_group_name, + self._workspace_name, + name, + parameters=SsoSetting(enable_sso=enable_sso), + ) + def _get_workspace_location(self) -> str: workspace = self._workspace_operations.get(self._resource_group_name, self._workspace_name) return str(workspace.location) From 723e26811828b005fcbf21a54c5006f59cce9ee3 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:45:53 +0530 Subject: [PATCH 16/28] adding change log entry --- sdk/ml/azure-ai-ml/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 7f90463f43ed..bd5086c2b383 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.19.0 (unreleased) +### Feature Added + + - Added enable_sso operation under compute operation that will allow user to enable sso setting of a compute instance without any write permission set on compute. + ### Bugs Fixed - Workspace update no longer broken for older workspaces due to deprecated tags. - Support credential-less fileshare datastore From f29cb6297e297bccc4ea55918b1b8ad4ad4456e8 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Fri, 26 Jul 2024 09:37:03 +0530 Subject: [PATCH 17/28] addressing review comment --- .../azure-ai-ml/azure/ai/ml/operations/_compute_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py index 57b53fa10132..d7a7ddcf1e8b 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py @@ -424,7 +424,7 @@ def list_sizes(self, *, location: Optional[str] = None, compute_type: Optional[s @distributed_trace @monitor_with_activity(ops_logger, "Compute.enablesso", ActivityType.PUBLICAPI) @experimental - def enable_sso(self, *, name: str, enable_sso: Optional[bool] = True): + def enable_sso(self, *, name: str, enable_sso: bool = True): """enable sso for a compute instance. :keyword name: Name of the compute instance. From 296bb47f5751be9a9cd013fe9c6c2fcd9ad4a53e Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:32:24 +0530 Subject: [PATCH 18/28] fixing ut --- .../tests/compute/unittests/test_compute_operations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/tests/compute/unittests/test_compute_operations.py b/sdk/ml/azure-ai-ml/tests/compute/unittests/test_compute_operations.py index da5b971dda07..6640c5704ed1 100644 --- a/sdk/ml/azure-ai-ml/tests/compute/unittests/test_compute_operations.py +++ b/sdk/ml/azure-ai-ml/tests/compute/unittests/test_compute_operations.py @@ -3,7 +3,6 @@ from unittest.mock import Mock import pytest -from azure.identity import DefaultAzureCredential from pytest_mock import MockFixture from azure.ai.ml import load_compute @@ -17,6 +16,7 @@ ) from azure.ai.ml.operations import ComputeOperations from azure.ai.ml.operations._local_job_invoker import CommonRuntimeHelper +from azure.identity import DefaultAzureCredential @pytest.fixture @@ -24,11 +24,13 @@ def mock_compute_operation( mock_workspace_scope: OperationScope, mock_operation_config: OperationConfig, mock_aml_services_2022_10_01_preview: Mock, + mock_aml_services_2023_04_01_preview: Mock, ) -> ComputeOperations: yield ComputeOperations( operation_scope=mock_workspace_scope, operation_config=mock_operation_config, service_client=mock_aml_services_2022_10_01_preview, + service_client_2024=mock_aml_services_2023_04_01_preview, ) From 5e147e7e64e2890a6ba7482a96388b9a4a3feef9 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:19:00 +0530 Subject: [PATCH 19/28] addressing api review comment --- .../azure-ai-ml/azure/ai/ml/operations/_compute_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py index d7a7ddcf1e8b..4ac8b55a93e2 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_compute_operations.py @@ -424,7 +424,7 @@ def list_sizes(self, *, location: Optional[str] = None, compute_type: Optional[s @distributed_trace @monitor_with_activity(ops_logger, "Compute.enablesso", ActivityType.PUBLICAPI) @experimental - def enable_sso(self, *, name: str, enable_sso: bool = True): + def enable_sso(self, *, name: str, enable_sso: bool = True) -> None: """enable sso for a compute instance. :keyword name: Name of the compute instance. From d452f7fe49825f3f027fb460e64ebda2f4dddd48 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:37:02 +0530 Subject: [PATCH 20/28] adding a fix for subnet validation in compute instance --- .../azure/ai/ml/entities/_compute/compute_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py index 2fa8e06054a4..3f9112efa9d0 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py @@ -344,7 +344,7 @@ def _to_dict(self) -> Dict: return res def _set_full_subnet_name(self, subscription_id: str, rg: str) -> None: - if self.network_settings: + if self.network_settings.vnet_name or self.network_settings.subnet: self.subnet = get_subnet_str( self.network_settings.vnet_name, self.network_settings.subnet, From c184b7554836db6ae0044549a0288604ca8f305a Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:03:56 +0530 Subject: [PATCH 21/28] my py fix --- .../azure/ai/ml/entities/_compute/compute_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py index 3f9112efa9d0..b47a41bdd6cb 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py @@ -344,7 +344,7 @@ def _to_dict(self) -> Dict: return res def _set_full_subnet_name(self, subscription_id: str, rg: str) -> None: - if self.network_settings.vnet_name or self.network_settings.subnet: + if self.network_settings and (self.network_settings.vnet_name or self.network_settings.subnet): self.subnet = get_subnet_str( self.network_settings.vnet_name, self.network_settings.subnet, From 8a4f0ae57ea8fe99317f54093621c70ffe83ad4b Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:48:02 +0530 Subject: [PATCH 22/28] fixing autoML serialization issue --- .../azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py index c79b1029eb51..aec3a69420dc 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/automl_job.py @@ -8,7 +8,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, Optional, Union -from azure.ai.ml._restclient.v2023_04_01_preview.models import ( +from azure.ai.ml._restclient.v2024_01_01_preview.models import ( JobBase, MLTableJobInput, QueueSettings, From e25d3644257b7073e09c3395f9329812569c0bc8 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:19:45 +0530 Subject: [PATCH 23/28] formatting --- .../unittests/test_automl_image_classification.py | 4 ++-- .../test_automl_image_classification_multilabel.py | 4 ++-- .../test_automl_image_instance_segmentation.py | 10 +++++----- .../unittests/test_automl_image_object_detection.py | 10 +++++----- .../automl_job/unittests/test_automl_image_schema.py | 2 +- .../automl_job/unittests/test_automl_nlp_schema.py | 12 ++++++------ .../unittests/test_automl_tabular_schema.py | 4 ++-- .../automl_job/unittests/test_classification_job.py | 3 ++- .../automl_job/unittests/test_forecasting_job.py | 2 +- .../automl_job/unittests/test_regression_job.py | 2 +- .../unittests/test_text_classification_job.py | 2 +- .../test_text_classification_multilabel_job.py | 2 +- .../tests/automl_job/unittests/test_text_ner_job.py | 2 +- 13 files changed, 30 insertions(+), 29 deletions(-) diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification.py index 168dfa9e99f3..bae4aa823a96 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification.py @@ -5,14 +5,14 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import ( +from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2024_01_01_preview.models import ( ClassificationPrimaryMetrics, LearningRateScheduler, MLTableJobInput, SamplingAlgorithmType, StochasticOptimizer, ) -from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity from azure.ai.ml.automl import image_classification from azure.ai.ml.constants._common import AssetTypes from azure.ai.ml.entities._inputs_outputs import Input diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification_multilabel.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification_multilabel.py index 0b4bd4ff5472..a04515626e26 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification_multilabel.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_classification_multilabel.py @@ -5,14 +5,14 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import ( +from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2024_01_01_preview.models import ( ClassificationMultilabelPrimaryMetrics, LearningRateScheduler, MLTableJobInput, SamplingAlgorithmType, StochasticOptimizer, ) -from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity from azure.ai.ml.automl import image_classification_multilabel from azure.ai.ml.constants._common import AssetTypes from azure.ai.ml.entities._inputs_outputs import Input diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py index f1a79f667b78..fa4d121358df 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py @@ -5,18 +5,18 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import ( +from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType +from azure.ai.ml._restclient.v2024_01_01_preview.models import ( InstanceSegmentationPrimaryMetrics, LearningRateScheduler, + LogTrainingMetrics, + LogValidationLoss, MLTableJobInput, ModelSize, SamplingAlgorithmType, StochasticOptimizer, - LogTrainingMetrics, - LogValidationLoss, ) -from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity -from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType from azure.ai.ml.automl import image_instance_segmentation from azure.ai.ml.constants._common import AssetTypes from azure.ai.ml.entities._inputs_outputs import Input diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py index 87e13be22453..75c6cab1be94 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py @@ -5,18 +5,18 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import ( +from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType +from azure.ai.ml._restclient.v2024_01_01_preview.models import ( LearningRateScheduler, + LogTrainingMetrics, + LogValidationLoss, MLTableJobInput, ModelSize, ObjectDetectionPrimaryMetrics, SamplingAlgorithmType, StochasticOptimizer, - LogTrainingMetrics, - LogValidationLoss, ) -from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity -from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType from azure.ai.ml.automl import image_object_detection from azure.ai.ml.constants._common import AssetTypes from azure.ai.ml.entities._inputs_outputs import Input diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py index bc0bc554fda8..28bf73c60e0c 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py @@ -48,7 +48,7 @@ ImageObjectDetection as RestImageObjectDetection, ) from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ImageSweepSettings as RestImageSweepSettings -from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( +from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import ( InstanceSegmentationPrimaryMetrics, JobBase, LogVerbosity, diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_nlp_schema.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_nlp_schema.py index 220c9c643667..90af08f318ca 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_nlp_schema.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_nlp_schema.py @@ -8,12 +8,6 @@ from azure.ai.ml import load_job from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import AutoMLJob as RestAutoMLJob from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import BanditPolicy as RestBanditPolicy -from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( - ClassificationPrimaryMetrics, - JobBase, - LogVerbosity, - MLTableJobInput, -) from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import NlpFixedParameters as RestNlpFixedParameters from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( NlpParameterSubspace as RestNlpParameterSubspace, @@ -30,6 +24,12 @@ TextClassificationMultilabel as RestTextClassificationMultilabel, ) from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import TextNer as RestTextNer +from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import ( + ClassificationPrimaryMetrics, + JobBase, + LogVerbosity, + MLTableJobInput, +) from azure.ai.ml._scope_dependent_operations import OperationScope from azure.ai.ml._utils.utils import dump_yaml_to_file, load_yaml, to_iso_duration_format_mins from azure.ai.ml.automl import ( diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_tabular_schema.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_tabular_schema.py index b167cf3bf111..9104a6095c72 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_tabular_schema.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_tabular_schema.py @@ -26,7 +26,6 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( ForecastingTrainingSettings as RestForecastingTrainingSettings, ) -from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import JobBase, LogVerbosity, MLTableJobInput from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import Regression as RestRegression from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import RegressionPrimaryMetrics from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( @@ -38,6 +37,7 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models._models_py3 import ( TableVerticalLimitSettings as RestTableVerticalLimitSettings, ) +from azure.ai.ml._restclient.v2024_01_01_preview.models._models_py3 import JobBase, LogVerbosity, MLTableJobInput from azure.ai.ml._schema.automl.table_vertical.regression import AutoMLRegressionSchema from azure.ai.ml._scope_dependent_operations import OperationScope from azure.ai.ml._utils.utils import camel_to_snake, to_iso_duration_format_mins @@ -47,11 +47,11 @@ from azure.ai.ml.entities import Job from azure.ai.ml.entities._inputs_outputs import Input from azure.ai.ml.entities._job.automl.automl_job import AutoMLJob -from azure.ai.ml.exceptions import ValidationException from azure.ai.ml.entities._job.automl.tabular.featurization_settings import TabularFeaturizationSettings from azure.ai.ml.entities._job.automl.tabular.forecasting_settings import ForecastingSettings from azure.ai.ml.entities._job.automl.tabular.limit_settings import TabularLimitSettings from azure.ai.ml.entities._job.automl.training_settings import TrainingSettings +from azure.ai.ml.exceptions import ValidationException @pytest.fixture(autouse=True) diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_classification_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_classification_job.py index 81c3b6830c28..bd745980479a 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_classification_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_classification_job.py @@ -6,8 +6,9 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import CustomNCrossValidations, MLTableJobInput +from azure.ai.ml._restclient.v2023_04_01_preview.models import CustomNCrossValidations from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml.automl import ClassificationModels, ClassificationPrimaryMetrics, classification from azure.ai.ml.constants import TabularTrainingMode from azure.ai.ml.constants._common import AssetTypes diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_forecasting_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_forecasting_job.py index 7f8bd606ed39..442dda027af1 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_forecasting_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_forecasting_job.py @@ -5,8 +5,8 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import MLTableJobInput from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml.automl import ( ForecastingModels, ForecastingPrimaryMetrics, diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_regression_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_regression_job.py index 48ab6d5baa24..58c387b1b8e4 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_regression_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_regression_job.py @@ -5,8 +5,8 @@ import pytest from azure.ai.ml import UserIdentityConfiguration -from azure.ai.ml._restclient.v2023_04_01_preview.models import MLTableJobInput from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml.automl import RegressionModels, RegressionPrimaryMetrics, regression from azure.ai.ml.constants import TabularTrainingMode from azure.ai.ml.constants._common import AssetTypes diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_job.py index ff342614b4af..57cf6badb6f6 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_job.py @@ -6,7 +6,6 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models import ( JobBase, LogVerbosity, - MLTableJobInput, NlpFixedParameters, NlpParameterSubspace, NlpSweepSettings, @@ -18,6 +17,7 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models._azure_machine_learning_workspaces_enums import ( ClassificationPrimaryMetrics, ) +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml._utils.utils import to_iso_duration_format_mins from azure.ai.ml.automl import text_classification from azure.ai.ml.constants._common import AssetTypes diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_multilabel_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_multilabel_job.py index 26a37067b961..f0f3e1bac816 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_multilabel_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_classification_multilabel_job.py @@ -8,7 +8,6 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models import ( JobBase, LogVerbosity, - MLTableJobInput, NlpFixedParameters, NlpParameterSubspace, NlpSweepSettings, @@ -21,6 +20,7 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models._azure_machine_learning_workspaces_enums import ( ClassificationPrimaryMetrics, ) +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml._utils.utils import to_iso_duration_format_mins from azure.ai.ml.automl import text_classification_multilabel from azure.ai.ml.constants._common import AssetTypes diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_ner_job.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_ner_job.py index b05669988abf..2a9a608f7bee 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_ner_job.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_text_ner_job.py @@ -6,7 +6,6 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models import ( JobBase, LogVerbosity, - MLTableJobInput, NlpFixedParameters, NlpParameterSubspace, NlpSweepSettings, @@ -20,6 +19,7 @@ from azure.ai.ml._restclient.v2023_04_01_preview.models._azure_machine_learning_workspaces_enums import ( ClassificationPrimaryMetrics, ) +from azure.ai.ml._restclient.v2024_01_01_preview.models import MLTableJobInput from azure.ai.ml._utils.utils import to_iso_duration_format_mins from azure.ai.ml.automl import text_ner from azure.ai.ml.constants._common import AssetTypes From 9624cfaaf261da1c773466b5c00fe3244c3fab43 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:51:20 +0530 Subject: [PATCH 24/28] moving auto ML job create API to 2024 --- sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py index cbcbb09ede68..ed9e5fb5fbce 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py @@ -725,7 +725,8 @@ def _create_or_update_with_different_version_api( # pylint: disable=name-too-lo service_client_operation = self.service_client_01_2024_preview.jobs if rest_job_resource.properties.job_type == RestJobType.PIPELINE: service_client_operation = self.service_client_01_2024_preview.jobs - + if rest_job_resource.properties.job_type == RestJobType.AUTO_ML: + service_client_operation = self.service_client_01_2024_preview.jobs if rest_job_resource.properties.job_type == RestJobType.SWEEP: service_client_operation = self.service_client_01_2024_preview.jobs From 4811abcef2a57dc637894846e866183216beda36 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:23:05 +0530 Subject: [PATCH 25/28] adding compliant logger --- .../azure/ai/ml/_logging/compliant_logger.py | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py new file mode 100644 index 000000000000..5f5d5ab3e6a6 --- /dev/null +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py @@ -0,0 +1,178 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +""" This is logger utility which will work with allowed logged filter AML policy + https://github.com/Azure/azure-policy/blob/master/built-in-policies/policyDefinitions/Machine%20Learning/AllowedLogFilter_EnforceSetting.json + You have to define the same "logFilters" while initializing the logger using "enable_compliant_logging" method + e.g. + log filters: ["^SystemLog:.*$"] + initialize : enable_compliant_logging(format_key="prefix", + format_key_value="SystemLog", + format=f"%(prefix)s{logging.BASIC_FORMAT}") + By default log message will not compliant e.g. not modified +""" + +import logging +import sys +from datetime import datetime +from threading import Lock +from typing import Optional + +_LOCK = Lock() +_FORMAT_KEY = None +_FORMAT_VALUE = None + + +def set_format_key(key_name: str) -> None: + with _LOCK: + _FORMAT_KEY = key_name + + +def get_format_key() -> Optional[str]: + return _FORMAT_KEY + + +def set_format_value(value: str) -> None: + with _LOCK: + _FORMAT_VALUE = value + + +def get_format_value() -> Optional[str]: + return _FORMAT_VALUE + + +def get_default_logging_format() -> str: + return f"%({get_format_key()})s{logging.BASIC_FORMAT}" + + +class CompliantLogger(logging.getLoggerClass()): + """ + Subclass of the default logging class with an explicit `is_record_compliant` parameter + on all logging methods. It will pass an `extra` param with `format` key + (value depending on whether `is_record_compliant` is True or False) to the + handlers. + + The default value for data `is_record_compliant` is `False` for all methods. + + Implementation is inspired by: + https://github.com/python/cpython/blob/3.8/Lib/logging/__init__.py + """ + + def __init__(self, name: str, handlers=None): + super().__init__(name) # type: ignore + + self.format_key = get_format_key() + self.format_value = get_format_value() + + if handlers: + self.handlers = handlers + + self.start_time = datetime.now() + self.metric_count = 1 + # number of iterable items that are logged + self.max_iter_items = 10 + + def _log( + self, + level, + msg, + args=None, + exc_info=None, + extra=None, + stack_info=False, + stacklevel=1, + is_record_compliant=False, + ): + if is_record_compliant: + format_value = self.format_value + else: + format_value = "" + + if extra: + extra.update({self.format_key: format_value}) + else: + extra = {self.format_key: format_value} + + if sys.version_info[1] <= 7: + super(CompliantLogger, self)._log( + level=level, + msg=msg, + args=args, + exc_info=exc_info, + extra=extra, + stack_info=stack_info, + ) + else: + super(CompliantLogger, self)._log( + level=level, + msg=msg, + args=args, + exc_info=exc_info, + extra=extra, + stack_info=stack_info, + stacklevel=stacklevel, # type: ignore + ) + + +_logging_basic_config_set_warning = """ +******************************************************************************** +The root logger already has handlers set! As a result, the behavior of this +library is undefined. If running in Python >= 3.8, this library will attempt to +call logging.basicConfig(force=True), which will remove all existing root +handlers. See https://stackoverflow.com/q/20240464 and +https://github.com/Azure/confidential-ml-utils/issues/33 for more information. +******************************************************************************** +""" + + +def enable_compliant_logging( + format_key: str = "prefix", + format_key_value: str = "SystemLog:", + **kwargs, +) -> None: + """ + The default format is `logging.BASIC_FORMAT` (`%(levelname)s:%(name)s:%(message)s`). + All other kwargs are passed to `logging.basicConfig`. Sets the default + logger class and root logger to be compliant. This means the format + string `%(prefix)` will work. + + :param format_key: key for format + :type format_key: str + :param format_key_value: value for format + :type format_key_value: str + + Set the format using the `format` kwarg. + + If running in Python >= 3.8, will attempt to add `force=True` to the kwargs + for logging.basicConfig. + + The standard implementation of the logging API is a good reference: + https://github.com/python/cpython/blob/3.9/Lib/logging/__init__.py + """ + set_format_key(format_key) + set_format_value(format_key_value) + + if "format" not in kwargs: + kwargs["format"] = get_default_logging_format() + + # Ensure that all loggers created via `logging.getLogger` are instances of + # the `CompliantLogger` class. + logging.setLoggerClass(CompliantLogger) + + if len(logging.root.handlers) > 0: + p = get_format_value() + for line in _logging_basic_config_set_warning.splitlines(): + print(f"{p}{line}", file=sys.stderr) + + if "force" not in kwargs and sys.version_info >= (3, 8): + kwargs["force"] = True + + root = CompliantLogger(logging.root.name, handlers=logging.root.handlers) + + logging.root = root + logging.Logger.root = root # type: ignore + logging.Logger.manager = logging.Manager(root) # type: ignore + + # https://github.com/kivy/kivy/issues/6733 + logging.basicConfig(**kwargs) From 03ee8f6e742651f286d56105c59b9ae069b432f6 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:10:06 +0530 Subject: [PATCH 26/28] getting compliant logger fix --- .../azure/ai/ml/_logging/compliant_logger.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py index 5f5d5ab3e6a6..179aa9e63f99 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py @@ -24,20 +24,19 @@ _FORMAT_VALUE = None -def set_format_key(key_name: str) -> None: +# pylint: disable=global-statement +def set_format(key_name: str, value: str) -> None: with _LOCK: + global _FORMAT_KEY _FORMAT_KEY = key_name + global _FORMAT_VALUE + _FORMAT_VALUE = value def get_format_key() -> Optional[str]: return _FORMAT_KEY -def set_format_value(value: str) -> None: - with _LOCK: - _FORMAT_VALUE = value - - def get_format_value() -> Optional[str]: return _FORMAT_VALUE @@ -135,7 +134,7 @@ def enable_compliant_logging( The default format is `logging.BASIC_FORMAT` (`%(levelname)s:%(name)s:%(message)s`). All other kwargs are passed to `logging.basicConfig`. Sets the default logger class and root logger to be compliant. This means the format - string `%(prefix)` will work. + string `%(xxxx)` will work. :param format_key: key for format :type format_key: str @@ -150,8 +149,7 @@ def enable_compliant_logging( The standard implementation of the logging API is a good reference: https://github.com/python/cpython/blob/3.9/Lib/logging/__init__.py """ - set_format_key(format_key) - set_format_value(format_key_value) + set_format(format_key, format_key_value) if "format" not in kwargs: kwargs["format"] = get_default_logging_format() From ae9591a68e480f7e15fc46df628a22cc54791fc8 Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:27:21 +0530 Subject: [PATCH 27/28] better variable name for line is compliant --- .../azure/ai/ml/_logging/compliant_logger.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py index 179aa9e63f99..d2657a96cae2 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py @@ -47,12 +47,12 @@ def get_default_logging_format() -> str: class CompliantLogger(logging.getLoggerClass()): """ - Subclass of the default logging class with an explicit `is_record_compliant` parameter + Subclass of the default logging class with an explicit `is_compliant` parameter on all logging methods. It will pass an `extra` param with `format` key - (value depending on whether `is_record_compliant` is True or False) to the + (value depending on whether `is_compliant` is True or False) to the handlers. - The default value for data `is_record_compliant` is `False` for all methods. + The default value for data `is_compliant` is `False` for all methods. Implementation is inspired by: https://github.com/python/cpython/blob/3.8/Lib/logging/__init__.py @@ -81,9 +81,9 @@ def _log( extra=None, stack_info=False, stacklevel=1, - is_record_compliant=False, + is_compliant=False, ): - if is_record_compliant: + if is_compliant: format_value = self.format_value else: format_value = "" From 45c7515f253d98f5b686e70ce495a831bf76620c Mon Sep 17 00:00:00 2001 From: Amit Chauhan <70937115+achauhan-scc@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:49:23 +0530 Subject: [PATCH 28/28] fix mypy issue --- sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py index d2657a96cae2..bc339d609775 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_logging/compliant_logger.py @@ -45,7 +45,7 @@ def get_default_logging_format() -> str: return f"%({get_format_key()})s{logging.BASIC_FORMAT}" -class CompliantLogger(logging.getLoggerClass()): +class CompliantLogger(logging.getLoggerClass()): # type: ignore """ Subclass of the default logging class with an explicit `is_compliant` parameter on all logging methods. It will pass an `extra` param with `format` key