[Dpg] DPG integration (Azure#25397)

* add PyYAML

* code

* README.md format

* multi client

* code

* gen multi client

* single client gen

* compatible with meta storage code

* add log to lookup readme

* dpg package does not need _meta.json in MAINFEST.in

* add language, apiviewartifacts

* optimize

* Update setup.py

Author: msyyc

swagger_to_sdk_config_dpg.json

@@ -2,10 +2,7 @@
   "meta": {
     "autorest_options": {
       "version": "3.8.1",
-      "use": ["@autorest/[email protected]", "@autorest/[email protected]"],
-      "python": "",
-      "sdkrel:python-sdks-folder": "./sdk/.",
-      "version-tolerant": ""
+      "use": ["@autorest/[email protected]", "@autorest/[email protected]"]
     },
     "advanced_options": {
       "create_sdk_pull_requests": true,

tools/azure-sdk-tools/packaging_tools/auto_codegen.py

@@ -4,10 +4,10 @@
 from pathlib import Path
 from subprocess import check_call
 
-from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG
+from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE
 from .generate_sdk import generate
 from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview,
-                             format_samples)
+                             format_samples, gen_dpg)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -24,8 +24,11 @@ def main(generate_input, generate_output):
     for input_readme in data["relatedReadmeMdFiles"]:
         relative_path_readme = str(Path(spec_folder, input_readme))
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
-        config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG
-        config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag)
+        if 'resource-manager' in input_readme:
+            config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True,
+                              python_tag=python_tag)
+        else:
+            config = gen_dpg(input_readme, data.get('autorestConfig', ''))
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 
@@ -54,7 +57,7 @@ def main(generate_input, generate_output):
             try:
                 update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme)
             except Exception as e:
-                _LOGGER.info(str(e))
+                _LOGGER.info(f"fail to update meta: {str(e)}")
 
             # Setup package locally
             check_call(

tools/azure-sdk-tools/packaging_tools/auto_package.py

@@ -46,6 +46,11 @@ def main(generate_input, generate_output):
         # to distinguish with track1
         if 'azure-mgmt-' in package_name:
             package["packageName"] = "track2_" + package["packageName"]
+        for artifact in package["artifacts"]:
+            if ".whl" in artifact:
+                package["apiViewArtifact"] = artifact
+                package["language"] = "Python"
+                break
         result["packages"].append(package)
 
     with open(generate_output, "w") as writer:

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -7,15 +7,19 @@
 from azure_devtools.ci_tools.git_tools import get_add_diff_file_list
 from pathlib import Path
 from subprocess import check_call
-from typing import List
+from typing import List, Dict, Any
+from glob import glob
+import yaml
+
+from .swaggertosdk.autorest_tools import build_autorest_options, generate_code
+from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config
 
-from .swaggertosdk.autorest_tools import build_autorest_options
 
 _LOGGER = logging.getLogger(__name__)
 _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
 
 DEFAULT_DEST_FOLDER = "./dist"
-
+_DPG_README = "README.md"
 
 def get_package_names(sdk_folder):
     files = get_add_diff_file_list(sdk_folder)
@@ -42,9 +46,13 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name,
 
     readme_file = str(Path(spec_folder, input_readme))
     global_conf = config["meta"]
-    local_conf = config["projects"][readme_file]
+    local_conf = config.get("projects", {}).get(readme_file, {})
 
-    cmd = ["autorest", input_readme]
+    if "resource-manager" in input_readme:
+        cmd = ["autorest", input_readme]
+    else:
+        # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger
+        cmd = ["autorest", _DPG_README]
     cmd += build_autorest_options(global_conf, local_conf)
 
     # metadata
@@ -71,23 +79,24 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name,
     _LOGGER.info(f"Saved metadata to {metadata_file_path}")
 
     # Check whether MANIFEST.in includes _meta.json
-    require_meta = "include _meta.json\n"
-    manifest_file = os.path.join(package_folder, "MANIFEST.in")
-    if not os.path.exists(manifest_file):
-        _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
-        return
-
-    includes = []
-    write_flag = False
-    with open(manifest_file, "r") as f:
-        includes = f.readlines()
-        if require_meta not in includes:
-            includes = [require_meta] + includes
-            write_flag = True
-
-    if write_flag:
-        with open(manifest_file, "w") as f:
-            f.write("".join(includes))
+    if "resource-manager" in input_readme:
+        require_meta = "include _meta.json\n"
+        manifest_file = os.path.join(package_folder, "MANIFEST.in")
+        if not os.path.exists(manifest_file):
+            _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
+            return
+
+        includes = []
+        write_flag = False
+        with open(manifest_file, "r") as f:
+            includes = f.readlines()
+            if require_meta not in includes:
+                includes = [require_meta] + includes
+                write_flag = True
+
+        if write_flag:
+            with open(manifest_file, "w") as f:
+                f.write("".join(includes))
 
 
 # find all the files of one folder, including files in subdirectory
@@ -129,6 +138,154 @@ def judge_tag_preview(path: str) -> bool:
     return 'preview' in api_version
 
 
+def extract_yaml_content(autorest_config: str) -> str:
+    num = []
+    content = autorest_config.split('\r\n')
+    for i in range(len(content)):
+        if "```" in content[i]:
+            num.append(i)
+    if len(num) != 2:
+        raise Exception(f"autorestConfig content is not valid: {autorest_config}")
+    return '\n'.join(content[num[0] + 1: num[1]])
+
+
+def add_config_title(content: str) -> str:
+    return f"# autorest configuration for Python\n\n{content}"
+
+
+def yaml_block(content: str, annotation: str = "", tag: str = "") -> str:
+    annotation = f"{annotation}\n\n" if annotation else annotation
+    return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n"
+
+
+def gen_package_name(origin_config: Dict[str, Any]) -> str:
+    return Path(origin_config["output-folder"]).parts[-1]
+
+
+def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]:
+    return {
+        "package-name": gen_package_name(origin_config),
+        "license-header": "MICROSOFT_MIT_NO_VERSION",
+        "package-version": origin_config.get("package-version", "1.0.0b1"),
+        "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]],
+        "package-mode": "dataplane",
+        "output-folder": "../",
+    }
+
+
+def gen_general_namespace(package_name: str) -> str:
+    return package_name.replace('-', '.')
+
+
+def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str:
+    package_name = Path(origin_config["output-folder"]).parts[-1]
+    readme_config = gen_basic_config(origin_config)
+    readme_config.update({
+        "namespace": gen_general_namespace(package_name),
+    })
+    readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings")
+    return add_config_title(readme_content)
+
+
+def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]:
+    tag_config = {}
+    package_name = gen_package_name(origin_config)
+    for tag in origin_config["batch"]:
+        tag_name = tag["tag"]
+        extra_part = tag_name.split("-")[-1]
+        tag_config[tag_name] = {
+            "namespace": gen_general_namespace(package_name) + f".{extra_part}",
+        }
+
+    return tag_config
+
+
+def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]:
+    batch_config = []
+    for item in origin_config["batch"]:
+        for _, value in item.items():
+            batch_config.append({value: True})
+    return {"batch": batch_config}
+
+
+def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str:
+    # generate config
+    basic_config = gen_basic_config(origin_config)
+    batch_config = gen_batch_config(origin_config)
+    tag_config = gen_tag_config(origin_config)
+
+    # convert to string
+    readme_content = yaml_block(yaml.dump(basic_config), "### Settings")
+    readme_content += yaml_block(yaml.dump(batch_config), "\n### Python multi-client")
+    for tag, value in tag_config.items():
+        readme_content += yaml_block(
+            yaml.dump(value),
+            f"\n### Tag: {tag}",
+            f"$({tag})",
+        )
+
+    return add_config_title(readme_content)
+
+
+# generate swagger/README.md and return relative path based on SDK repo root path
+def gen_dpg_config(autorest_config: str) -> str:
+    # remove useless lines
+    autorest_config = extract_yaml_content(autorest_config)
+    _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}")
+
+    # make dir if not exist
+    origin_config = yaml.safe_load(autorest_config)
+    _LOGGER.info(f"autorestConfig: {origin_config}")
+    swagger_folder = str(Path(origin_config["output-folder"], "swagger"))
+    if not os.path.exists(swagger_folder):
+        os.makedirs(swagger_folder)
+
+    # generate autorest configuration
+    if "batch:" in autorest_config:
+        readme_content = gen_dpg_config_multi_client(origin_config)
+    else:
+        readme_content = gen_dpg_config_single_client(origin_config)
+
+    # output autorest configuration
+    swagger_readme = str(Path(swagger_folder, _DPG_README))
+    with open(swagger_readme, "w") as file:
+        file.write(readme_content)
+    return swagger_readme
+
+
+def lookup_swagger_readme(rest_readme_path: str) -> str:
+    all_swagger_readme = glob(str(Path(f'sdk/*/*/swagger/{_DPG_README}')))
+    for readme in all_swagger_readme:
+        with open(readme, 'r') as file:
+            content = file.read()
+            if rest_readme_path in content:
+                _LOGGER.info(f"find swagger readme: {readme}")
+                return readme
+    _LOGGER.info(f"do not find swagger readme which contains {rest_readme_path}")
+    return ""
+
+
+def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]:
+    # generate or find swagger/README.md
+    if autorest_config:
+        swagger_readme = gen_dpg_config(autorest_config)
+    else:
+        swagger_readme = lookup_swagger_readme(rest_readme_path)
+    if not swagger_readme:
+        return
+
+    # extract global config
+    global_config = read_config('.', CONFIG_FILE_DPG)
+
+    # generate code
+    current_path = os.getcwd()
+    os.chdir(Path(swagger_readme).parent)
+    generate_code(_DPG_README, global_config["meta"], {})
+    os.chdir(current_path)
+
+    return global_config
+
+
 def format_samples(sdk_code_path) -> None:
     generate_sample_path = Path(sdk_code_path) / 'generated_samples'
     if not generate_sample_path.exists():

tools/azure-sdk-tools/setup.py

@@ -25,6 +25,7 @@
     "azure-mgmt-storage",
     "azure-mgmt-keyvault",
     "python-dotenv",
+    "PyYAML"
 ]
 
 setup(