From 0854c6014589aba2e18d1381adeb16e07adbdfc1 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Fri, 28 Mar 2025 09:41:29 +0100 Subject: [PATCH 1/7] Added github action to update feature catalog file --- .github/scripts/update_feature_catalog.py | 79 +++++++++++++++++++ .../generate-feature-coverage-page.yml | 19 +++++ 2 files changed, 98 insertions(+) create mode 100644 .github/scripts/update_feature_catalog.py create mode 100644 .github/workflows/generate-feature-coverage-page.yml diff --git a/.github/scripts/update_feature_catalog.py b/.github/scripts/update_feature_catalog.py new file mode 100644 index 0000000000..05a5a82d7b --- /dev/null +++ b/.github/scripts/update_feature_catalog.py @@ -0,0 +1,79 @@ +import os +import sys + +import yaml + +DEFAULT_STATUS = 'not supported' +DEFAULT_EMULATION_LEVEL = 'CRUD' + +MD_FILE_HEADER = """--- +title: "AWS Service Feature Coverage" +linkTitle: "⭐ Feature Coverage" +weight: 1 +description: > + Overview of the implemented AWS APIs and their level of parity with the AWS cloud +aliases: + - /localstack/coverage/ + - /aws/feature-coverage/ +hide_readingtime: true +--- + + +## Emulation Levels + +* CRUD: The service accepts requests and returns proper (potentially static) responses. + No additional business logic besides storing entities. +* Emulated: The service imitates the functionality, including synchronous and asynchronous business logic operating on service entities. + +| Service / Feature | Implementation status | Emulation Level | Limitations | +|-------------------|----------------|-----------------|--------------------------| +""" + + +def yml_to_md_table(yml_content): + service_name = yml_content.get('name') + emulation_level = yml_content.get('emulation_level', DEFAULT_EMULATION_LEVEL) + md_table = f"| **{service_name}** | [Details 🔍] | {emulation_level} | |\n" + + # Add features + for feature in yml_content.get('features', []): + feature_name = feature.get('name', '') + documentation_page = feature.get('documentation_page') + if documentation_page: + feature_name = f'[{feature_name}]({documentation_page})' + status = feature.get('status', DEFAULT_STATUS) + + # Get limitations + limitations = feature.get('limitations', []) + limitations_md = '\n '.join(limitations) if limitations else '' + + md_table += f"| {feature_name} | {status} | {emulation_level} | {limitations_md} |\n" + + return md_table + +def load_yaml_file(file_path: str): + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except yaml.YAMLError as e: + print(f"Error parsing YAML file: {e}") + sys.exit(1) + except FileNotFoundError: + print(f"YAML file not found: {file_path}") + sys.exit(1) + +def main(): + changed_features_files = os.getenv('ALL_CHANGED_FEATURES_FILES').split(',') + try: + with open("new-feature-coverage.md", "w") as feature_coverage_md_file: + feature_coverage_md_file.write(MD_FILE_HEADER) + + for file_path in changed_features_files: + features_file = load_yaml_file(file_path) + features_md = yml_to_md_table(features_file) + feature_coverage_md_file.write(features_md) + except Exception as e: + print(f"Error writing to file: {e}") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/generate-feature-coverage-page.yml b/.github/workflows/generate-feature-coverage-page.yml new file mode 100644 index 0000000000..084b73bf53 --- /dev/null +++ b/.github/workflows/generate-feature-coverage-page.yml @@ -0,0 +1,19 @@ +name: Update Feature catalog docs +on: + schedule: + - cron: 0 5 * * MON # TO-DO define schedule + workflow_dispatch: +jobs: + update-feature-catalog: + name: Update Feature catalog docs + runs-on: ubuntu-latest + steps: + - name: Checkout community repository + uses: actions/checkout@v4 + with: + repository: 'localstack/localstack' + ref: 'master' + path: 'localstack-community' + + - name: Generate feature catalog file from feature files + run: python3 .github/scripts/update_feature_catalog.py From 2d2a84db5f04b825b90d5689d6eb0f16145f0260 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:25:16 +0200 Subject: [PATCH 2/7] Added gh action to generate feature catalog page and create PR --- .../workflows/docs-update-feature-catalog.yml | 48 +++++++++++++++++++ .../generate-feature-coverage-page.yml | 19 -------- .../generate_feature_catalog_page.py | 0 3 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/docs-update-feature-catalog.yml delete mode 100644 .github/workflows/generate-feature-coverage-page.yml rename .github/scripts/update_feature_catalog.py => scripts/generate_feature_catalog_page.py (100%) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml new file mode 100644 index 0000000000..3cce51af27 --- /dev/null +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -0,0 +1,48 @@ +name: Update feature catalog page +on: + schedule: + - cron: 0 10 * * MON + workflow_dispatch: +jobs: + generate-feature-catalog-file: + name: Generate feature catalog file + runs-on: ubuntu-latest + steps: + - name: Checkout docs repository + uses: actions/checkout@v4 + with: + repository: 'localstack/docs' + path: 'localstack-docs' + + - name: Download features files from Collect feature files (GitHub) + uses: actions/download-artifact@v4 + with: + path: features-files-community + name: features-files + repository: localstack/localstack + +# - name: Download features files from Collect feature files from PRO (GitHub) +# uses: actions/download-artifact@v4 +# with: +# path: features-files-ext +# name: features-files-ext +# repository: localstack/localstack +# github-token: ${{ secrets.GH_PAT }} # token with actions:read permissions on target repo + + - name: Generate feature catalog page + run: python3 scripts/generate_feature_catalog_page.py + + pr-with-updated-feature-catalog: + name: Create PR with updated feature catalog file + runs-on: ubuntu-latest + steps: + - name: Create PR + uses: peter-evans/create-pull-request@v7 + with: + title: "Update Feature catalog page" + body: "This PR updates Feature catalog page based on feature catalog YAML files" + branch: "update-feature-catalog" + author: "LocalStack Bot <localstack-bot@users.noreply.github.com>" + committer: "LocalStack Bot <localstack-bot@users.noreply.github.com>" + commit-message: "Upgrade feature catalog" + labels: "documentation" diff --git a/.github/workflows/generate-feature-coverage-page.yml b/.github/workflows/generate-feature-coverage-page.yml deleted file mode 100644 index 084b73bf53..0000000000 --- a/.github/workflows/generate-feature-coverage-page.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Update Feature catalog docs -on: - schedule: - - cron: 0 5 * * MON # TO-DO define schedule - workflow_dispatch: -jobs: - update-feature-catalog: - name: Update Feature catalog docs - runs-on: ubuntu-latest - steps: - - name: Checkout community repository - uses: actions/checkout@v4 - with: - repository: 'localstack/localstack' - ref: 'master' - path: 'localstack-community' - - - name: Generate feature catalog file from feature files - run: python3 .github/scripts/update_feature_catalog.py diff --git a/.github/scripts/update_feature_catalog.py b/scripts/generate_feature_catalog_page.py similarity index 100% rename from .github/scripts/update_feature_catalog.py rename to scripts/generate_feature_catalog_page.py From 967cb9eb9512d9306496175620bc7f3844649606 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Thu, 3 Apr 2025 12:28:44 +0200 Subject: [PATCH 3/7] Script to generate md file from ext and community feature files --- .../workflows/docs-update-feature-catalog.yml | 24 ++--- scripts/generate_feature_catalog_page.py | 95 +++++++++++++------ 2 files changed, 78 insertions(+), 41 deletions(-) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml index 3cce51af27..90b51587ba 100644 --- a/.github/workflows/docs-update-feature-catalog.yml +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -12,7 +12,6 @@ jobs: uses: actions/checkout@v4 with: repository: 'localstack/docs' - path: 'localstack-docs' - name: Download features files from Collect feature files (GitHub) uses: actions/download-artifact@v4 @@ -21,27 +20,28 @@ jobs: name: features-files repository: localstack/localstack -# - name: Download features files from Collect feature files from PRO (GitHub) -# uses: actions/download-artifact@v4 -# with: -# path: features-files-ext -# name: features-files-ext -# repository: localstack/localstack -# github-token: ${{ secrets.GH_PAT }} # token with actions:read permissions on target repo + - name: Download features files from Collect feature files from PRO (GitHub) + uses: actions/download-artifact@v4 + with: + path: features-files-ext + name: features-files-ext + repository: localstack/localstack + github-token: ${{ secrets.GH_AC_UPDATE_FEATURE_CATALOG }} - name: Generate feature catalog page run: python3 scripts/generate_feature_catalog_page.py + env: + PATH_FEATURE_FILES_COMMUNITY: 'features-files-community' + PATH_FEATURE_FILES_EXT: 'features-files-ext' + PATH_FEATURE_CATALOG_MD: 'content/en/user-guide/aws/feature-coverage.md' - pr-with-updated-feature-catalog: - name: Create PR with updated feature catalog file - runs-on: ubuntu-latest - steps: - name: Create PR uses: peter-evans/create-pull-request@v7 with: title: "Update Feature catalog page" body: "This PR updates Feature catalog page based on feature catalog YAML files" branch: "update-feature-catalog" + add-paths: 'content/en/user-guide/aws/feature-coverage.md' author: "LocalStack Bot <localstack-bot@users.noreply.github.com>" committer: "LocalStack Bot <localstack-bot@users.noreply.github.com>" commit-message: "Upgrade feature catalog" diff --git a/scripts/generate_feature_catalog_page.py b/scripts/generate_feature_catalog_page.py index 05a5a82d7b..1863a9948e 100644 --- a/scripts/generate_feature_catalog_page.py +++ b/scripts/generate_feature_catalog_page.py @@ -1,10 +1,12 @@ import os import sys +from pathlib import Path import yaml DEFAULT_STATUS = 'not supported' DEFAULT_EMULATION_LEVEL = 'CRUD' +FEATURES_FILE_NAME='features.yml' MD_FILE_HEADER = """--- title: "AWS Service Feature Coverage" @@ -26,30 +28,40 @@ * Emulated: The service imitates the functionality, including synchronous and asynchronous business logic operating on service entities. | Service / Feature | Implementation status | Emulation Level | Limitations | -|-------------------|----------------|-----------------|--------------------------| -""" +|-------------------|----------------|-----------------|--------------------------|""" +class FeatureCatalogMarkdownGenerator: + md_content = [MD_FILE_HEADER] -def yml_to_md_table(yml_content): - service_name = yml_content.get('name') - emulation_level = yml_content.get('emulation_level', DEFAULT_EMULATION_LEVEL) - md_table = f"| **{service_name}** | [Details 🔍] | {emulation_level} | |\n" + def __init__(self, file_path: str): + self.file_path = file_path + pass - # Add features - for feature in yml_content.get('features', []): - feature_name = feature.get('name', '') - documentation_page = feature.get('documentation_page') - if documentation_page: - feature_name = f'[{feature_name}]({documentation_page})' - status = feature.get('status', DEFAULT_STATUS) + def add_service_section(self, feature_file_content: str): + service_name = feature_file_content.get('name') + emulation_level = feature_file_content.get('emulation_level', DEFAULT_EMULATION_LEVEL) + self.md_content.append(f"| **{service_name}** | [Details 🔍] | {emulation_level} | |") - # Get limitations - limitations = feature.get('limitations', []) - limitations_md = '\n '.join(limitations) if limitations else '' + def add_features_rows(self, feature_file_content: str): + for feature in feature_file_content.get('features', []): + feature_name = feature.get('name', '') + documentation_page = feature.get('documentation_page') + if documentation_page: + feature_name = f'[{feature_name}]({documentation_page})' + status = feature.get('status', DEFAULT_STATUS) - md_table += f"| {feature_name} | {status} | {emulation_level} | {limitations_md} |\n" + limitations = feature.get('limitations', []) + limitations_md = '\n '.join(limitations) if limitations else '' - return md_table + self.md_content.append(f"| {feature_name} | {status} | | {limitations_md} |") + + def generate_file(self): + try: + with open(self.file_path, "w") as feature_coverage_md_file: + feature_coverage_md_file.writelines(s + '\n' for s in self.md_content) + except Exception as e: + print(f"Error writing to file: {e}") + sys.exit(1) def load_yaml_file(file_path: str): try: @@ -62,18 +74,43 @@ def load_yaml_file(file_path: str): print(f"YAML file not found: {file_path}") sys.exit(1) +def get_service_path_to_abs_community_ext_paths(community_files_path: str, ext_files_path: str) -> dict[str, (str, str)]: + relative_to_abs_paths = {} + for community_abs_path in Path(community_files_path).rglob(FEATURES_FILE_NAME): + rel_path = str(community_abs_path.relative_to(community_files_path)) + relative_to_abs_paths[rel_path] = (community_abs_path, None) + + for abs_path_ext in Path(ext_files_path).rglob(FEATURES_FILE_NAME): + rel_path = str(abs_path_ext.relative_to(ext_files_path)) + if rel_path in relative_to_abs_paths: + community_abs_path, _ = relative_to_abs_paths[rel_path] + relative_to_abs_paths[rel_path] = (community_abs_path, abs_path_ext) + else: + relative_to_abs_paths[rel_path] = (None, abs_path_ext) + return relative_to_abs_paths + def main(): - changed_features_files = os.getenv('ALL_CHANGED_FEATURES_FILES').split(',') - try: - with open("new-feature-coverage.md", "w") as feature_coverage_md_file: - feature_coverage_md_file.write(MD_FILE_HEADER) - - for file_path in changed_features_files: - features_file = load_yaml_file(file_path) - features_md = yml_to_md_table(features_file) - feature_coverage_md_file.write(features_md) - except Exception as e: - print(f"Error writing to file: {e}") + community_feature_files_path = os.getenv('PATH_FEATURE_FILES_COMMUNITY') + ext_feature_files_path = os.getenv('PATH_FEATURE_FILES_EXT') + feature_catalog_md_file_path = os.getenv('PATH_FEATURE_CATALOG_MD') + + service_path_to_abs_paths = get_service_path_to_abs_community_ext_paths(community_feature_files_path, ext_feature_files_path) + md_generator = FeatureCatalogMarkdownGenerator(feature_catalog_md_file_path) + + for service_name in sorted(service_path_to_abs_paths): + abs_path_community, abs_path_ext = service_path_to_abs_paths.get(service_name) + service_definition_created = False + if abs_path_community: + feature_file_community = load_yaml_file(abs_path_community) + md_generator.add_service_section(feature_file_community) + service_definition_created = True + md_generator.add_features_rows(feature_file_community) + if abs_path_ext: + feature_file_ext = load_yaml_file(abs_path_ext) + if not service_definition_created: + md_generator.add_service_section(feature_file_community) + md_generator.add_features_rows(feature_file_ext) + md_generator.generate_file() if __name__ == "__main__": main() From 9f2226f81aae5e172fcc2fd9bcb379842a45a4fe Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Thu, 3 Apr 2025 12:42:35 +0200 Subject: [PATCH 4/7] Removed unnecessary repo name --- .github/workflows/docs-update-feature-catalog.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml index 90b51587ba..8f02bd4f2e 100644 --- a/.github/workflows/docs-update-feature-catalog.yml +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -10,8 +10,6 @@ jobs: steps: - name: Checkout docs repository uses: actions/checkout@v4 - with: - repository: 'localstack/docs' - name: Download features files from Collect feature files (GitHub) uses: actions/download-artifact@v4 @@ -41,7 +39,7 @@ jobs: title: "Update Feature catalog page" body: "This PR updates Feature catalog page based on feature catalog YAML files" branch: "update-feature-catalog" - add-paths: 'content/en/user-guide/aws/feature-coverage.md' + add-paths: "content/en/user-guide/aws/feature-coverage.md" author: "LocalStack Bot <localstack-bot@users.noreply.github.com>" committer: "LocalStack Bot <localstack-bot@users.noreply.github.com>" commit-message: "Upgrade feature catalog" From b76f2a57d5ee2d7943ff50b2ffd34640365f68cd Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:27:22 +0200 Subject: [PATCH 5/7] Added steps to detect GH action run-id --- .../workflows/docs-update-feature-catalog.yml | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml index 8f02bd4f2e..0c46c2e65b 100644 --- a/.github/workflows/docs-update-feature-catalog.yml +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -5,18 +5,38 @@ on: workflow_dispatch: jobs: generate-feature-catalog-file: - name: Generate feature catalog file + name: Generate feature catalog page runs-on: ubuntu-latest steps: - name: Checkout docs repository uses: actions/checkout@v4 + - name: Latest run-id from community repository + run: | + latest_workflow_id=$(curl -s https://api.github.com/repos/localstack/localstack/actions/workflows \ + | jq '.workflows[] | select(.name=="Artifact with features files").id') + latest_run_id=$(curl -s \ + https://api.github.com/repos/localstack/localstack/actions/workflows/$latest_workflow_id/runs | jq '.workflow_runs[0].id') + echo "Latest run-id: ${latest_run_id}" + echo "FEATURES_ARTIFACTS_COMMUNITY_RUN_ID=${latest_run_id}" >> $GITHUB_ENV + - name: Download features files from Collect feature files (GitHub) uses: actions/download-artifact@v4 with: path: features-files-community name: features-files + github-token: ${{ secrets.GH_PAT_FEATURE_CATALOG_PAGE }} # PAT with access to artifacts from GH Actions repository: localstack/localstack + run-id: ${{ env.FEATURES_ARTIFACTS_COMMUNITY_RUN_ID }} + + - name: Latest run-id from ext repository + run: | + latest_workflow_id=$(curl -s https://api.github.com/repos/localstack/localstack-ext/actions/workflows \ + | jq '.workflows[] | select(.name=="Artifact with features files").id') + latest_run_id=$(curl -s \ + https://api.github.com/repos/localstack/localstack-ext/actions/workflows/$latest_workflow_id/runs | jq '.workflow_runs[0].id') + echo "Latest run-id: ${latest_run_id}" + echo "FEATURES_ARTIFACTS_EXT_RUN_ID=${latest_run_id}" >> $GITHUB_ENV - name: Download features files from Collect feature files from PRO (GitHub) uses: actions/download-artifact@v4 @@ -24,7 +44,8 @@ jobs: path: features-files-ext name: features-files-ext repository: localstack/localstack - github-token: ${{ secrets.GH_AC_UPDATE_FEATURE_CATALOG }} + github-token: ${{ secrets.GH_PAT_FEATURE_CATALOG_PAGE_PRO }} # PAT with access to artifacts from GH Actions + run-id: ${{ env.FEATURES_ARTIFACTS_EXT_RUN_ID }} - name: Generate feature catalog page run: python3 scripts/generate_feature_catalog_page.py From f0f7d430694924a2dfa2ce4ea5d64a95da5a343e Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Mon, 7 Apr 2025 17:49:41 +0200 Subject: [PATCH 6/7] Fixed comments from pr --- .github/workflows/docs-update-feature-catalog.yml | 6 +++--- scripts/generate_feature_catalog_page.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml index 0c46c2e65b..eb4d1400ee 100644 --- a/.github/workflows/docs-update-feature-catalog.yml +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -1,7 +1,7 @@ name: Update feature catalog page on: schedule: - - cron: 0 10 * * MON + - cron: 0 10 * * Tue workflow_dispatch: jobs: generate-feature-catalog-file: @@ -14,7 +14,7 @@ jobs: - name: Latest run-id from community repository run: | latest_workflow_id=$(curl -s https://api.github.com/repos/localstack/localstack/actions/workflows \ - | jq '.workflows[] | select(.name=="Artifact with features files").id') + | jq '.workflows[] | select(.name=="AWS / Archive feature files").id') latest_run_id=$(curl -s \ https://api.github.com/repos/localstack/localstack/actions/workflows/$latest_workflow_id/runs | jq '.workflow_runs[0].id') echo "Latest run-id: ${latest_run_id}" @@ -32,7 +32,7 @@ jobs: - name: Latest run-id from ext repository run: | latest_workflow_id=$(curl -s https://api.github.com/repos/localstack/localstack-ext/actions/workflows \ - | jq '.workflows[] | select(.name=="Artifact with features files").id') + | jq '.workflows[] | select(.name=="AWS / Archive feature files").id') latest_run_id=$(curl -s \ https://api.github.com/repos/localstack/localstack-ext/actions/workflows/$latest_workflow_id/runs | jq '.workflow_runs[0].id') echo "Latest run-id: ${latest_run_id}" diff --git a/scripts/generate_feature_catalog_page.py b/scripts/generate_feature_catalog_page.py index 1863a9948e..a8b03dcaaf 100644 --- a/scripts/generate_feature_catalog_page.py +++ b/scripts/generate_feature_catalog_page.py @@ -4,7 +4,7 @@ import yaml -DEFAULT_STATUS = 'not supported' +DEFAULT_STATUS = 'unsupported' DEFAULT_EMULATION_LEVEL = 'CRUD' FEATURES_FILE_NAME='features.yml' @@ -68,10 +68,10 @@ def load_yaml_file(file_path: str): with open(file_path, 'r') as file: return yaml.safe_load(file) except yaml.YAMLError as e: - print(f"Error parsing YAML file: {e}") + sys.stdout.write(f"::error title=Failed to parse features file::An error occurred while parsing {file_path}: {e}") sys.exit(1) except FileNotFoundError: - print(f"YAML file not found: {file_path}") + sys.stdout.write(f"::error title=Missing features file::No features file found at {file_path}") sys.exit(1) def get_service_path_to_abs_community_ext_paths(community_files_path: str, ext_files_path: str) -> dict[str, (str, str)]: From 3cd0cb4969300f639d47ca9fe8b7a558af4b61e5 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Mon, 7 Apr 2025 17:57:19 +0200 Subject: [PATCH 7/7] Upper-cased abbr for cron job definition --- .github/workflows/docs-update-feature-catalog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-update-feature-catalog.yml b/.github/workflows/docs-update-feature-catalog.yml index eb4d1400ee..23539e8069 100644 --- a/.github/workflows/docs-update-feature-catalog.yml +++ b/.github/workflows/docs-update-feature-catalog.yml @@ -1,7 +1,7 @@ name: Update feature catalog page on: schedule: - - cron: 0 10 * * Tue + - cron: 0 10 * * TUE workflow_dispatch: jobs: generate-feature-catalog-file: