From 62b39292c360ac7b6a2f4777f2aa2ac202fd2582 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 8 Jun 2022 15:08:52 +0800 Subject: [PATCH 01/13] add PyYAML --- tools/azure-sdk-tools/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 53c644ce32ba..e22499c6995b 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -24,6 +24,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", + "PyYAML==6.0" ] setup( From 146c007a7b335bf58f086d6a0c83413646512621 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 17:54:37 +0800 Subject: [PATCH 02/13] code --- .../packaging_tools/auto_codegen.py | 7 +- .../packaging_tools/generate_utils.py | 75 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index dc19dca8128b..7e693c277fe3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg _LOGGER = logging.getLogger(__name__) @@ -23,7 +23,10 @@ def main(generate_input, generate_output): relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + else: + generate_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 574d7b23eb6d..f414be22851f 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,9 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List +from typing import List, Union +from glob import glob +import yaml from .swaggertosdk.autorest_tools import build_autorest_options @@ -126,3 +128,74 @@ def judge_tag_preview(path: str) -> bool: _LOGGER.info(f'find single api version:{api_version}') return 'preview' in api_version + + +def extract_yaml_content(autorest_config: str) -> str: + num = [] + content = autorest_config.split('\r\n') + for i in range(len(content)): + if "```" in content[i]: + num.append(i) + if len(num) != 2: + raise Exception(f"autorestConfig content is not valid: {autorest_config}") + return '\n'.join(content[num[0] + 1: num[1]]) + + +def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: + return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + + +def generate_dpg_config(autorest_config: str) -> str: + # remove useless lines + autorest_config = extract_yaml_content(autorest_config) + _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + + # make dir if not exist + origin_config = yaml.safe_load(autorest_config) + _LOGGER.info(f"autoresConfig: {origin_config}") + swagger_folder = str(Path(origin_config["output-folder"], "swagger")) + if not os.path.exists(swagger_folder): + os.makedirs(swagger_folder) + + # generate autorest configuration + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_content = { + "package-name": package_name, + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": "1.0.0b1", + "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "output-folder": "../" + package_name.replace('-', '/'), + "namespace": package_name.replace('-', '.') + } + + # output autorest configuration + swagger_readme = str(Path(swagger_folder, "README.md")) + with open(swagger_readme, "w") as file: + file.write(add_yaml_title(yaml.safe_dump(readme_content))) + return swagger_readme + + +def lookup_swagger_readme(rest_readme_path: str) -> str: + all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + for readme in all_swagger_readme: + with open(readme, 'r') as file: + content = file.read() + if rest_readme_path in content: + return readme + return "" + + +def generate_dpg(rest_readme_path: str, autorest_config: str): + if autorest_config: + swagger_readme = generate_dpg_config(autorest_config) + else: + swagger_readme = lookup_swagger_readme(rest_readme_path) + if not swagger_readme: + return + + # extract global config + + # generate code From 8edbe377abe12d8ccdc3d1e8f851681e1e9e6b71 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 18:06:28 +0800 Subject: [PATCH 03/13] README.md format --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index f414be22851f..9c5ed347d46b 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -142,7 +142,9 @@ def extract_yaml_content(autorest_config: str) -> str: def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + if annotation: + annotation = f"{annotation}\n\n" + return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" def generate_dpg_config(autorest_config: str) -> str: @@ -166,7 +168,7 @@ def generate_dpg_config(autorest_config: str) -> str: "no-namespace-folders": True, "version-tolerant": True, "package-version": "1.0.0b1", - "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], "output-folder": "../" + package_name.replace('-', '/'), "namespace": package_name.replace('-', '.') } From ced64c782278a5154722fc8122fb9f801bf38a5c Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 11:41:16 +0800 Subject: [PATCH 04/13] multi client --- .../packaging_tools/auto_codegen.py | 4 +- .../packaging_tools/generate_utils.py | 111 ++++++++++++++---- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 7e693c277fe3..9f2196cd6c58 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg _LOGGER = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - generate_dpg(input_readme, data.get('autorestConfig', '')) + gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 9c5ed347d46b..2658920aba67 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,7 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List, Union +from typing import List, Dict, Any from glob import glob import yaml @@ -141,42 +141,107 @@ def extract_yaml_content(autorest_config: str) -> str: return '\n'.join(content[num[0] + 1: num[1]]) -def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - if annotation: - annotation = f"{annotation}\n\n" - return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" +def add_config_title(content: str) -> str: + return f"# autorest configuration for Python\n\n{content}" -def generate_dpg_config(autorest_config: str) -> str: +def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: + annotation = f"{annotation}\n\n" if annotation else annotation + return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" + + + + + +def gen_package_name(origin_config: Dict[str, Any]) -> str: + return Path(origin_config["output-folder"]).parts[-1] + + +def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + return { + "package-name": gen_package_name(origin_config), + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": origin_config.get("package-version", "1.0.0b1"), + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + } + + +def gen_general_output_folder(package_name: str) -> str: + return "../" + package_name.replace('-', '/') + + +def gen_general_namespace(package_name: str) -> str: + return package_name.replace('-', '.') + + +def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_config = gen_basic_config(origin_config) + readme_config.update({ + "output-folder": gen_general_output_folder(package_name), + "namespace": gen_general_namespace(package_name), + }) + readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") + return add_config_title(readme_content) + + +def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + tag_config = {} + package_name = gen_package_name(origin_config) + for tag in origin_config["batch"]: + tag_name = tag["tag"] + extra_part = tag_name.split("-")[-1] + tag_config[tag_name] = { + "namespace": gen_general_namespace(package_name) + f".{extra_part}", + "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" + } + + return tag_config + + +def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: + # generate config + basic_config = gen_basic_config(origin_config) + batch_config = {"batch": origin_config["batch"]} + tag_config = gen_tag_config(origin_config) + + # convert to string + readme_content = yaml_block(yaml.dump(basic_config), "### Settings") + readme_content += yaml_block(yaml.dump(batch_config), "\n### Python multi-client") + for tag, value in tag_config.items(): + readme_content += yaml_block( + yaml.dump(value), + f"\n### Tag: {tag}", + f"$(tag) == '{tag}'", + ) + + return add_config_title(readme_content) + +def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) - _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") # make dir if not exist origin_config = yaml.safe_load(autorest_config) - _LOGGER.info(f"autoresConfig: {origin_config}") + _LOGGER.info(f"autorestConfig: {origin_config}") swagger_folder = str(Path(origin_config["output-folder"], "swagger")) if not os.path.exists(swagger_folder): os.makedirs(swagger_folder) # generate autorest configuration - package_name = Path(origin_config["output-folder"]).parts[-1] - readme_content = { - "package-name": package_name, - "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, - "package-version": "1.0.0b1", - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], - "output-folder": "../" + package_name.replace('-', '/'), - "namespace": package_name.replace('-', '.') - } + if "batch:" in autorest_config: + readme_content = gen_dpg_config_multi_client(origin_config) + else: + readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration swagger_readme = str(Path(swagger_folder, "README.md")) with open(swagger_readme, "w") as file: - file.write(add_yaml_title(yaml.safe_dump(readme_content))) + file.write(readme_content) return swagger_readme @@ -190,9 +255,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def generate_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str): if autorest_config: - swagger_readme = generate_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: From 0a29f58ac1515265150756cce3633600962ae85a Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 13:42:49 +0800 Subject: [PATCH 05/13] code --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 2658920aba67..1342d96f04e8 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -150,9 +150,6 @@ def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" - - - def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] @@ -217,9 +214,10 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: f"\n### Tag: {tag}", f"$(tag) == '{tag}'", ) - + return add_config_title(readme_content) + def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) From 409ac725fabcf718d4782c2951c2746db0ca2607 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:10:10 +0800 Subject: [PATCH 06/13] gen multi client --- .../packaging_tools/generate_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 1342d96f04e8..7f0dfc2b130d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -199,10 +199,18 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return tag_config +def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + batch_config = [] + for item in origin_config["batch"]: + for _, value in item.items(): + batch_config.append({value: True}) + return {"batch": batch_config} + + def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate config basic_config = gen_basic_config(origin_config) - batch_config = {"batch": origin_config["batch"]} + batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) # convert to string @@ -212,7 +220,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: readme_content += yaml_block( yaml.dump(value), f"\n### Tag: {tag}", - f"$(tag) == '{tag}'", + f"$({tag})", ) return add_config_title(readme_content) From f20bea0578718b1f1f64187381346b7b3d14761c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:36:12 +0800 Subject: [PATCH 07/13] single client gen --- swagger_to_sdk_config_dpg.json | 6 ++---- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 3 +-- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/swagger_to_sdk_config_dpg.json b/swagger_to_sdk_config_dpg.json index 4900e744c9e8..1d011e81afdc 100644 --- a/swagger_to_sdk_config_dpg.json +++ b/swagger_to_sdk_config_dpg.json @@ -1,10 +1,8 @@ { "meta": { "autorest_options": { - "version": "3.7.2", - "use": ["@autorest/python@5.16.0", "@autorest/modelerfour@4.19.3"], - "python": "", - "sdkrel:python-sdks-folder": "./sdk/.", + "version": "3.8.1", + "use": ["@autorest/python@5.17.0", "@autorest/modelerfour@4.23.5"], "version-tolerant": "" }, "advanced_options": { diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 9f2196cd6c58..d97a7891b518 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -4,7 +4,7 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg @@ -22,7 +22,6 @@ def main(generate_input, generate_output): for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 7f0dfc2b130d..8c05bc5ab505 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -10,7 +10,9 @@ from glob import glob import yaml -from .swaggertosdk.autorest_tools import build_autorest_options +from .swaggertosdk.autorest_tools import build_autorest_options, generate_code +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config + _LOGGER = logging.getLogger(__name__) _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) @@ -226,6 +228,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: return add_config_title(readme_content) +# generate swagger/README.md and return relative path based on SDK repo root path def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) @@ -262,6 +265,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: def gen_dpg(rest_readme_path: str, autorest_config: str): + # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) else: @@ -270,5 +274,10 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config + global_config = read_config('.', CONFIG_FILE_DPG)["meta"] # generate code + current_path = os.getcwd() + os.chdir(Path(swagger_readme).parent) + generate_code(Path(swagger_readme).parts[-1], global_config, {}) + os.chdir(current_path) \ No newline at end of file From 95fe92e3ccfc462c5c4f6fce2d15776e0baf7eb8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:02:00 +0800 Subject: [PATCH 08/13] compatible with meta storage code --- .../packaging_tools/auto_codegen.py | 4 ++-- .../packaging_tools/generate_utils.py | 24 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index d97a7891b518..b84696648979 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -25,7 +25,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -53,7 +53,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 8c05bc5ab505..4d8798417834 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -18,7 +18,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" - +_DPG_README = "README.md" def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) @@ -45,9 +45,13 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, readme_file = str(Path(spec_folder, input_readme)) global_conf = config["meta"] - local_conf = config["projects"][readme_file] + local_conf = config.get("projects", {}).get(readme_file, {}) - cmd = ["autorest", input_readme] + if "resource-manager" in input_readme: + cmd = ["autorest", input_readme] + else: + # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger + cmd = ["autorest", _DPG_README] cmd += build_autorest_options(global_conf, local_conf) # metadata @@ -248,14 +252,14 @@ def gen_dpg_config(autorest_config: str) -> str: readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration - swagger_readme = str(Path(swagger_folder, "README.md")) + swagger_readme = str(Path(swagger_folder, _DPG_README)) with open(swagger_readme, "w") as file: file.write(readme_content) return swagger_readme def lookup_swagger_readme(rest_readme_path: str) -> str: - all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + all_swagger_readme = glob(str(Path(f'sdk/*/*/swagger/{_DPG_README}'))) for readme in all_swagger_readme: with open(readme, 'r') as file: content = file.read() @@ -264,7 +268,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) @@ -274,10 +278,12 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config - global_config = read_config('.', CONFIG_FILE_DPG)["meta"] + global_config = read_config('.', CONFIG_FILE_DPG) # generate code current_path = os.getcwd() os.chdir(Path(swagger_readme).parent) - generate_code(Path(swagger_readme).parts[-1], global_config, {}) - os.chdir(current_path) \ No newline at end of file + generate_code(_DPG_README, global_config["meta"], {}) + os.chdir(current_path) + + return global_config From b99079e06da472c220c0dbfd3a7014e38ac9d892 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:33:53 +0800 Subject: [PATCH 09/13] add log to lookup readme --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 4d8798417834..ced287a096ba 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -264,7 +264,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: with open(readme, 'r') as file: content = file.read() if rest_readme_path in content: + _LOGGER.info(f"find swagger readme: {readme}") return readme + _LOGGER.info(f"do not find swagger readme which contains {rest_readme_path}") return "" From 3f0e9ed0b35f543ea9b8b26bc60496bedb74f45b Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 17:10:31 +0800 Subject: [PATCH 10/13] dpg package does not need _meta.json in MAINFEST.in --- .../packaging_tools/generate_utils.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index ced287a096ba..999aad2e7d87 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -78,23 +78,24 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, _LOGGER.info(f"Saved metadata to {metadata_file_path}") # Check whether MANIFEST.in includes _meta.json - require_meta = "include _meta.json\n" - manifest_file = os.path.join(package_folder, "MANIFEST.in") - if not os.path.exists(manifest_file): - _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") - return - - includes = [] - write_flag = False - with open(manifest_file, "r") as f: - includes = f.readlines() - if require_meta not in includes: - includes = [require_meta] + includes - write_flag = True - - if write_flag: - with open(manifest_file, "w") as f: - f.write("".join(includes)) + if "resource-manager" in input_readme: + require_meta = "include _meta.json\n" + manifest_file = os.path.join(package_folder, "MANIFEST.in") + if not os.path.exists(manifest_file): + _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") + return + + includes = [] + write_flag = False + with open(manifest_file, "r") as f: + includes = f.readlines() + if require_meta not in includes: + includes = [require_meta] + includes + write_flag = True + + if write_flag: + with open(manifest_file, "w") as f: + f.write("".join(includes)) # find all the files of one folder, including files in subdirectory From fb2a89dbd5c88580a8efad2eaccc6bda60447266 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 13 Jun 2022 15:22:36 +0800 Subject: [PATCH 11/13] add language, apiviewartifacts --- tools/azure-sdk-tools/packaging_tools/auto_package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py index 15e2c9090453..df1750003af3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_package.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_package.py @@ -46,6 +46,11 @@ def main(generate_input, generate_output): # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break result["packages"].append(package) with open(generate_output, "w") as writer: From c8877f643bbf55f499369edbd46237fe4af9cbd1 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:06:44 +0800 Subject: [PATCH 12/13] optimize --- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 065e569e4b69..c565d7f0520e 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -166,18 +166,13 @@ def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, "package-version": origin_config.get("package-version", "1.0.0b1"), "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "package-mode": "dataplane", + "output-folder": "../", } -def gen_general_output_folder(package_name: str) -> str: - return "../" + package_name.replace('-', '/') - - def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') @@ -186,7 +181,6 @@ def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] readme_config = gen_basic_config(origin_config) readme_config.update({ - "output-folder": gen_general_output_folder(package_name), "namespace": gen_general_namespace(package_name), }) readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") @@ -201,7 +195,6 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: extra_part = tag_name.split("-")[-1] tag_config[tag_name] = { "namespace": gen_general_namespace(package_name) + f".{extra_part}", - "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" } return tag_config From 15f033c0acf2854633d3affae2fd9ad90b4f54ff Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 26 Jul 2022 11:38:23 +0800 Subject: [PATCH 13/13] Update setup.py --- tools/azure-sdk-tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index a568c5f5cac5..db116f4616f0 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -25,7 +25,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", - "PyYAML==6.0" + "PyYAML" ] setup(