diff --git a/swagger_to_sdk_config_dpg.json b/swagger_to_sdk_config_dpg.json index f797c4f563c6..19c3eae36424 100644 --- a/swagger_to_sdk_config_dpg.json +++ b/swagger_to_sdk_config_dpg.json @@ -2,10 +2,7 @@ "meta": { "autorest_options": { "version": "3.8.1", - "use": ["@autorest/python@6.0.0", "@autorest/modelerfour@4.23.5"], - "python": "", - "sdkrel:python-sdks-folder": "./sdk/.", - "version-tolerant": "" + "use": ["@autorest/python@6.0.0", "@autorest/modelerfour@4.23.5"] }, "advanced_options": { "create_sdk_pull_requests": true, diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 460d9a546f60..14f87f40a813 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -4,10 +4,10 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples) + format_samples, gen_dpg) _LOGGER = logging.getLogger(__name__) @@ -24,8 +24,11 @@ def main(generate_input, generate_output): for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) + else: + config = gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -54,7 +57,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py index 15e2c9090453..df1750003af3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_package.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_package.py @@ -46,6 +46,11 @@ def main(generate_input, generate_output): # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break result["packages"].append(package) with open(generate_output, "w") as writer: diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 1990df93dcc8..c565d7f0520e 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -7,15 +7,19 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List +from typing import List, Dict, Any +from glob import glob +import yaml + +from .swaggertosdk.autorest_tools import build_autorest_options, generate_code +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config -from .swaggertosdk.autorest_tools import build_autorest_options _LOGGER = logging.getLogger(__name__) _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" - +_DPG_README = "README.md" def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) @@ -42,9 +46,13 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, readme_file = str(Path(spec_folder, input_readme)) global_conf = config["meta"] - local_conf = config["projects"][readme_file] + local_conf = config.get("projects", {}).get(readme_file, {}) - cmd = ["autorest", input_readme] + if "resource-manager" in input_readme: + cmd = ["autorest", input_readme] + else: + # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger + cmd = ["autorest", _DPG_README] cmd += build_autorest_options(global_conf, local_conf) # metadata @@ -71,23 +79,24 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, _LOGGER.info(f"Saved metadata to {metadata_file_path}") # Check whether MANIFEST.in includes _meta.json - require_meta = "include _meta.json\n" - manifest_file = os.path.join(package_folder, "MANIFEST.in") - if not os.path.exists(manifest_file): - _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") - return - - includes = [] - write_flag = False - with open(manifest_file, "r") as f: - includes = f.readlines() - if require_meta not in includes: - includes = [require_meta] + includes - write_flag = True - - if write_flag: - with open(manifest_file, "w") as f: - f.write("".join(includes)) + if "resource-manager" in input_readme: + require_meta = "include _meta.json\n" + manifest_file = os.path.join(package_folder, "MANIFEST.in") + if not os.path.exists(manifest_file): + _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") + return + + includes = [] + write_flag = False + with open(manifest_file, "r") as f: + includes = f.readlines() + if require_meta not in includes: + includes = [require_meta] + includes + write_flag = True + + if write_flag: + with open(manifest_file, "w") as f: + f.write("".join(includes)) # find all the files of one folder, including files in subdirectory @@ -129,6 +138,154 @@ def judge_tag_preview(path: str) -> bool: return 'preview' in api_version +def extract_yaml_content(autorest_config: str) -> str: + num = [] + content = autorest_config.split('\r\n') + for i in range(len(content)): + if "```" in content[i]: + num.append(i) + if len(num) != 2: + raise Exception(f"autorestConfig content is not valid: {autorest_config}") + return '\n'.join(content[num[0] + 1: num[1]]) + + +def add_config_title(content: str) -> str: + return f"# autorest configuration for Python\n\n{content}" + + +def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: + annotation = f"{annotation}\n\n" if annotation else annotation + return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" + + +def gen_package_name(origin_config: Dict[str, Any]) -> str: + return Path(origin_config["output-folder"]).parts[-1] + + +def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + return { + "package-name": gen_package_name(origin_config), + "license-header": "MICROSOFT_MIT_NO_VERSION", + "package-version": origin_config.get("package-version", "1.0.0b1"), + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "package-mode": "dataplane", + "output-folder": "../", + } + + +def gen_general_namespace(package_name: str) -> str: + return package_name.replace('-', '.') + + +def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_config = gen_basic_config(origin_config) + readme_config.update({ + "namespace": gen_general_namespace(package_name), + }) + readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") + return add_config_title(readme_content) + + +def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + tag_config = {} + package_name = gen_package_name(origin_config) + for tag in origin_config["batch"]: + tag_name = tag["tag"] + extra_part = tag_name.split("-")[-1] + tag_config[tag_name] = { + "namespace": gen_general_namespace(package_name) + f".{extra_part}", + } + + return tag_config + + +def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + batch_config = [] + for item in origin_config["batch"]: + for _, value in item.items(): + batch_config.append({value: True}) + return {"batch": batch_config} + + +def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: + # generate config + basic_config = gen_basic_config(origin_config) + batch_config = gen_batch_config(origin_config) + tag_config = gen_tag_config(origin_config) + + # convert to string + readme_content = yaml_block(yaml.dump(basic_config), "### Settings") + readme_content += yaml_block(yaml.dump(batch_config), "\n### Python multi-client") + for tag, value in tag_config.items(): + readme_content += yaml_block( + yaml.dump(value), + f"\n### Tag: {tag}", + f"$({tag})", + ) + + return add_config_title(readme_content) + + +# generate swagger/README.md and return relative path based on SDK repo root path +def gen_dpg_config(autorest_config: str) -> str: + # remove useless lines + autorest_config = extract_yaml_content(autorest_config) + _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") + + # make dir if not exist + origin_config = yaml.safe_load(autorest_config) + _LOGGER.info(f"autorestConfig: {origin_config}") + swagger_folder = str(Path(origin_config["output-folder"], "swagger")) + if not os.path.exists(swagger_folder): + os.makedirs(swagger_folder) + + # generate autorest configuration + if "batch:" in autorest_config: + readme_content = gen_dpg_config_multi_client(origin_config) + else: + readme_content = gen_dpg_config_single_client(origin_config) + + # output autorest configuration + swagger_readme = str(Path(swagger_folder, _DPG_README)) + with open(swagger_readme, "w") as file: + file.write(readme_content) + return swagger_readme + + +def lookup_swagger_readme(rest_readme_path: str) -> str: + all_swagger_readme = glob(str(Path(f'sdk/*/*/swagger/{_DPG_README}'))) + for readme in all_swagger_readme: + with open(readme, 'r') as file: + content = file.read() + if rest_readme_path in content: + _LOGGER.info(f"find swagger readme: {readme}") + return readme + _LOGGER.info(f"do not find swagger readme which contains {rest_readme_path}") + return "" + + +def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: + # generate or find swagger/README.md + if autorest_config: + swagger_readme = gen_dpg_config(autorest_config) + else: + swagger_readme = lookup_swagger_readme(rest_readme_path) + if not swagger_readme: + return + + # extract global config + global_config = read_config('.', CONFIG_FILE_DPG) + + # generate code + current_path = os.getcwd() + os.chdir(Path(swagger_readme).parent) + generate_code(_DPG_README, global_config["meta"], {}) + os.chdir(current_path) + + return global_config + + def format_samples(sdk_code_path) -> None: generate_sample_path = Path(sdk_code_path) / 'generated_samples' if not generate_sample_path.exists(): diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index f04d6eb63994..db116f4616f0 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -25,6 +25,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", + "PyYAML" ] setup(