From 14463bf8b4547a8acf596e1ef42c2ae735f7fad5 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:07:03 +0800 Subject: [PATCH 1/2] code --- scripts/auto_release/main.py | 4 +- scripts/automation_generate.sh | 4 +- .../packaging_tools/auto_codegen.py | 135 ------------------ .../packaging_tools/auto_package.py | 94 ------------ .../packaging_tools/sdk_generator.py | 127 ++++++++-------- .../packaging_tools/sdk_package.py | 5 +- tools/azure-sdk-tools/setup.py | 2 - 7 files changed, 75 insertions(+), 296 deletions(-) delete mode 100644 tools/azure-sdk-tools/packaging_tools/auto_codegen.py delete mode 100644 tools/azure-sdk-tools/packaging_tools/auto_package.py diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 934c567ae843..f285528e5e08 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -199,13 +199,13 @@ def generate_code(self): # generate code(be careful about the order) print_exec('python scripts/dev_setup.py -p azure-core') - print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') + print_check(f'python -m packaging_tools.sdk_generator {self.autorest_result} {self.autorest_result}') generate_result = self.get_autorest_result() self.tag_is_stable = list(generate_result.values())[0]['tagIsStable'] log(f"tag_is_stable is {self.tag_is_stable}") - print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') + print_check(f'python -m packaging_tools.sdk_package {self.autorest_result} {self.autorest_result}') def get_package_name_with_autorest_result(self): generate_result = self.get_autorest_result() diff --git a/scripts/automation_generate.sh b/scripts/automation_generate.sh index b568239b269f..a725dd314249 100644 --- a/scripts/automation_generate.sh +++ b/scripts/automation_generate.sh @@ -12,7 +12,7 @@ export PATH=/usr/local/n/versions/node/14.15.0/bin:$PATH TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json" # generate code -python -m packaging_tools.auto_codegen "$1" "$TEMP_FILE" 2>&1 +python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" 2>&1 echo "[Generate] codegen done!!!" if [ ! -f "$TEMP_FILE" ]; then echo "[Autorest]$TEMP_FILE does not exist!!!Error happened during codegen" @@ -20,7 +20,7 @@ if [ ! -f "$TEMP_FILE" ]; then fi # package -python -m packaging_tools.auto_package "$TEMP_FILE" "$2" 2>&1 +python -m packaging_tools.sdk_package "$TEMP_FILE" "$2" 2>&1 echo "[Generate] generate done!!!" if [ ! -f "$2" ]; then echo "[Autorest]$2 does not exist!!!Error happened during package" diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py deleted file mode 100644 index 28a0c53fee0d..000000000000 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ /dev/null @@ -1,135 +0,0 @@ -import argparse -import json -import logging -from pathlib import Path -from subprocess import check_call - -from .swaggertosdk.SwaggerToSdkCore import ( - CONFIG_FILE, -) -from .generate_sdk import generate -from .generate_utils import ( - get_package_names, - init_new_service, - update_servicemetadata, - judge_tag_preview, - format_samples, - gen_dpg, - dpg_relative_folder, -) - -_LOGGER = logging.getLogger(__name__) - - -def main(generate_input, generate_output): - with open(generate_input, "r") as reader: - data = json.load(reader) - - spec_folder = data["specFolder"] - sdk_folder = "." - result = {} - python_tag = data.get("python_tag") - package_total = set() - for input_readme in data["relatedReadmeMdFiles"]: - relative_path_readme = str(Path(spec_folder, input_readme)) - _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - if "resource-manager" in input_readme: - config = generate( - CONFIG_FILE, - sdk_folder, - [], - relative_path_readme, - spec_folder, - force_generation=True, - python_tag=python_tag, - ) - else: - config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder)) - package_names = get_package_names(sdk_folder) - _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") - - for folder_name, package_name in package_names: - if package_name in package_total: - continue - - package_total.add(package_name) - sdk_code_path = str(Path(sdk_folder, folder_name, package_name)) - if package_name not in result: - package_entry = {} - package_entry["packageName"] = package_name - package_entry["path"] = [folder_name] - package_entry["readmeMd"] = [input_readme] - package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) - result[package_name] = package_entry - else: - result[package_name]["path"].append(folder_name) - result[package_name]["readmeMd"].append(input_readme) - - # Generate some necessary file for new service - init_new_service(package_name, folder_name) - format_samples(sdk_code_path) - - # Update metadata - try: - update_servicemetadata( - sdk_folder, - data, - config, - folder_name, - package_name, - spec_folder, - input_readme, - ) - except Exception as e: - _LOGGER.info(f"fail to update meta: {str(e)}") - - # Setup package locally - check_call( - f"pip install --ignore-requires-python -e {sdk_code_path}", - shell=True, - ) - - # remove duplicates - for value in result.values(): - value["path"] = list(set(value["path"])) - value["readmeMd"] = list(set(value["readmeMd"])) - - with open(generate_output, "w") as writer: - json.dump(result, writer) - - -def generate_main(): - """Main method""" - - parser = argparse.ArgumentParser( - description="Build SDK using Autorest, offline version.", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument("generate_input", help="Generate input file path") - parser.add_argument("generate_output", help="Generate output file path") - parser.add_argument( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="Verbosity in INFO mode", - ) - parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode") - parser.add_argument( - "-c", - "--codegen", - dest="debug", - action="store_true", - help="Verbosity in DEBUG mode", - ) - - args = parser.parse_args() - main_logger = logging.getLogger() - logging.basicConfig() - main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO) - - main(args.generate_input, args.generate_output) - - -if __name__ == "__main__": - generate_main() diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py deleted file mode 100644 index 16ce57d08c19..000000000000 --- a/tools/azure-sdk-tools/packaging_tools/auto_package.py +++ /dev/null @@ -1,94 +0,0 @@ -import argparse -import json -import logging -import os -from pathlib import Path - -from azure_devtools.ci_tools.git_tools import get_diff_file_list -from .package_utils import create_package, change_log_generate, extract_breaking_change - -_LOGGER = logging.getLogger(__name__) - - -def main(generate_input, generate_output): - with open(generate_input, "r") as reader: - data = json.load(reader) - _LOGGER.info(f"auto_package input: {data}") - - sdk_folder = "." - result = {"packages": []} - for package in data.values(): - package_name = package["packageName"] - # Changelog - last_version = ["first release"] - if "azure-mgmt-" in package_name: - md_output = change_log_generate(package_name, last_version, package["tagIsStable"]) - else: - md_output = "data-plan skip changelog generation temporarily" - package["changelog"] = { - "content": md_output, - "hasBreakingChange": "Breaking Changes" in md_output, - "breakingChangeItems": extract_breaking_change(md_output), - } - package["version"] = last_version[-1] - - _LOGGER.info(f"[PACKAGE]({package_name})[CHANGELOG]:{md_output}") - # Built package - create_package(package_name) - folder_name = package["path"][0] - dist_path = Path(sdk_folder, folder_name, package_name, "dist") - package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)] - # Installation package - package["installInstructions"] = { - "full": "You can install the use using pip install of the artifacts.", - "lite": f"pip install {package_name}", - } - # to distinguish with track1 - if "azure-mgmt-" in package_name: - package["packageName"] = "track2_" + package["packageName"] - for artifact in package["artifacts"]: - if ".whl" in artifact: - package["apiViewArtifact"] = artifact - package["language"] = "Python" - break - result["packages"].append(package) - - with open(generate_output, "w") as writer: - json.dump(result, writer) - - -def generate_main(): - """Main method""" - - parser = argparse.ArgumentParser( - description="Build SDK using Autorest, offline version.", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument("generate_input", help="Generate input file path") - parser.add_argument("generate_output", help="Generate output file path") - parser.add_argument( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="Verbosity in INFO mode", - ) - parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode") - parser.add_argument( - "-c", - "--codegen", - dest="debug", - action="store_true", - help="Verbosity in DEBUG mode", - ) - - args = parser.parse_args() - main_logger = logging.getLogger() - logging.basicConfig() - main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO) - - main(args.generate_input, args.generate_output) - - -if __name__ == "__main__": - generate_main() diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index b31d7666d333..34bdf40321cd 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -2,10 +2,11 @@ import json import logging from pathlib import Path -import re from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE +from .swaggertosdk.SwaggerToSdkCore import ( + CONFIG_FILE, +) from .generate_sdk import generate from .generate_utils import ( get_package_names, @@ -14,6 +15,7 @@ judge_tag_preview, format_samples, gen_dpg, + dpg_relative_folder, ) _LOGGER = logging.getLogger(__name__) @@ -22,71 +24,80 @@ def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) - _LOGGER.info(f"auto_package input: {data}") + spec_folder = data["specFolder"] sdk_folder = "." result = {} python_tag = data.get("python_tag") package_total = set() - - # make sure spec_folder like: "../azure-rest-api-specs" - spec_folder = data["specFolder"] - input_readme = data["relatedReadmeMdFile"] - if "specification" in spec_folder: - spec_folder = str(Path(spec_folder.split("specification")[0])) - if "specification" not in input_readme: - input_readme = str(Path("specification") / input_readme) - _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - - if "resource-manager" in input_readme: - config = generate( - CONFIG_FILE, sdk_folder, [], input_readme, spec_folder, force_generation=True, python_tag=python_tag - ) + if "relatedReadmeMdFiles" in data: + readme_files = data["relatedReadmeMdFiles"] else: - config = gen_dpg(input_readme, data.get("autorestConfig", ""), spec_folder) - package_names = get_package_names(sdk_folder) - _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") - - for folder_name, package_name in package_names: - if package_name in package_total: - continue - - package_total.add(package_name) - sdk_code_path = str(Path(sdk_folder, folder_name, package_name)) - if package_name not in result: - package_entry = {} - package_entry["packageName"] = package_name - package_entry["path"] = [folder_name] - package_entry["readmeMd"] = [input_readme] - package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) - result[package_name] = package_entry - else: - result[package_name]["path"].append(folder_name) - result[package_name]["readmeMd"].append(input_readme) - - # Generate some necessary file for new service - init_new_service(package_name, folder_name) - format_samples(sdk_code_path) - - # Update metadata - try: - update_servicemetadata( + input_readme = data["relatedReadmeMdFile"] + if "specification" in spec_folder: + spec_folder = str(Path(spec_folder.split("specification")[0])) + if "specification" not in input_readme: + input_readme = str(Path("specification") / input_readme) + readme_files = [input_readme] + + for input_readme in readme_files: + relative_path_readme = str(Path(spec_folder, input_readme)) + _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") + if "resource-manager" in input_readme: + config = generate( + CONFIG_FILE, sdk_folder, - data, - config, - folder_name, - package_name, + [], + relative_path_readme, spec_folder, - input_readme, + force_generation=True, + python_tag=python_tag, + ) + else: + config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder)) + package_names = get_package_names(sdk_folder) + _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") + + for folder_name, package_name in package_names: + if package_name in package_total: + continue + + package_total.add(package_name) + sdk_code_path = str(Path(sdk_folder, folder_name, package_name)) + if package_name not in result: + package_entry = {} + package_entry["packageName"] = package_name + package_entry["path"] = [folder_name] + package_entry["readmeMd"] = [input_readme] + package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) + result[package_name] = package_entry + else: + result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) + + # Generate some necessary file for new service + init_new_service(package_name, folder_name) + format_samples(sdk_code_path) + + # Update metadata + try: + update_servicemetadata( + sdk_folder, + data, + config, + folder_name, + package_name, + spec_folder, + input_readme, + ) + except Exception as e: + _LOGGER.info(f"fail to update meta: {str(e)}") + + # Setup package locally + check_call( + f"pip install --ignore-requires-python -e {sdk_code_path}", + shell=True, ) - except Exception as e: - _LOGGER.info(f"fail to update meta: {str(e)}") - - # Setup package locally - check_call( - f"pip install --ignore-requires-python -e {sdk_code_path}", - shell=True, - ) # remove duplicates for value in result.values(): diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index 0a38cf870bdf..ba41737351d1 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -22,7 +22,7 @@ def main(generate_input, generate_output): # Changelog last_version = ["first release"] if "azure-mgmt-" in package_name: - md_output = change_log_generate(package_name, last_version) + md_output = change_log_generate(package_name, last_version, package["tagIsStable"]) else: md_output = "data-plan skip changelog generation temporarily" package["changelog"] = { @@ -41,7 +41,7 @@ def main(generate_input, generate_output): package["result"] = "succeeded" # Installation package package["installInstructions"] = { - "full": "You can install the use using pip install of the artificats.", + "full": "You can install the use using pip install of the artifacts.", "lite": f"pip install {package_name}", } # to distinguish with track1 @@ -52,7 +52,6 @@ def main(generate_input, generate_output): package["apiViewArtifact"] = artifact package["language"] = "Python" break - package["packageFolder"] = package["path"][0] result["packages"].append(package) with open(generate_output, "w") as writer: diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index ff723dc5231b..654964c80438 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -39,8 +39,6 @@ "console_scripts": [ "generate_package=packaging_tools.generate_package:generate_main", "generate_sdk=packaging_tools.generate_sdk:generate_main", - "auto_codegen=packaging_tools.auto_codegen:generate_main", - "auto_package=packaging_tools.auto_package:generate_main", "sdk_generator=packaging_tools.sdk_generator:generate_main", "sdk_package=packaging_tools.sdk_package:generate_main", "sdk_build=ci_tools.build:build", From 18b3571e88f9f7158eb30beddd6756571e9e92b7 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:27:51 +0800 Subject: [PATCH 2/2] code --- tools/azure-sdk-tools/packaging_tools/sdk_package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index ba41737351d1..78d9c869a51c 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -52,6 +52,7 @@ def main(generate_input, generate_output): package["apiViewArtifact"] = artifact package["language"] = "Python" break + package["packageFolder"] = package["path"][0] result["packages"].append(package) with open(generate_output, "w") as writer: