diff --git a/.github/features_schema.json b/.github/features_schema.json new file mode 100644 index 0000000..b5f030a --- /dev/null +++ b/.github/features_schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["service", "name", "features"], + "properties": { + "service": { + "type": "string", + "description": "Service's abbreviation in AWS" + }, + "name": { + "type": "string", + "description": "The display name of the service from AWS" + }, + "emulation_level": { + "type": "string", + "enum": ["CRUD", "emulated"], + "description": "The level of emulation support on the service level" + }, + "localstack_page": { + "type": "string", + "format": "uri", + "description": "URL to the LocalStack user guide documentation" + }, + "features": { + "type": "array", + "description": "List of features supported by the service", + "items": { + "type": "object", + "required": ["name", "description", "documentation_page", "status"], + "properties": { + "name": { + "type": "string", + "description": "Name of the feature" + }, + "description": { + "type": "string", + "description": "Description of the feature" + }, + "documentation_page": { + "type": "string", + "format": "uri", + "description": "URL to the AWS feature's documentation" + }, + "status": { + "type": "string", + "enum": ["supported", "unsupported"], + "description": "Current status of the feature" + }, + "api_methods": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of API methods associated with the feature" + }, + "limitations": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of known limitations for the feature" + }, + "emulation_level": { + "type": "string", + "enum": ["CRUD", "emulated"], + "description": "The level of emulation support" + } + }, + "additionalProperties": false + }, + "minItems": 0 + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/.github/scripts/check_features_files_exist.py b/.github/scripts/check_features_files_exist.py new file mode 100644 index 0000000..562923b --- /dev/null +++ b/.github/scripts/check_features_files_exist.py @@ -0,0 +1,44 @@ +import os +import sys + +FEATURES_FILE_NAME='features.yml' + +def find_paths_to_service_providers(directory: str) -> list[str]: + provider_path = os.path.join(directory, 'provider.py') + if os.path.isfile(provider_path): + return [provider_path] + paths = [] + for root, _, files in os.walk(directory): + if 'provider.py' in files: + paths.append(os.path.join(root, 'provider.py')) + return paths + + +def map_features_files_status(services_path, changed_files) -> dict[str, bool]: + features_files_exist_status = {} + for file_path in changed_files: + if file_path.startswith(services_path): + service_folder_name = file_path.removeprefix(services_path).split('/')[1] + service_path = os.path.join(services_path, service_folder_name) + + provider_files = find_paths_to_service_providers(service_path) + for provider_file in provider_files: + features_file = os.path.join(os.path.dirname(provider_file), FEATURES_FILE_NAME) + features_files_exist_status[features_file] = os.path.exists(features_file) + return features_files_exist_status + +def main(): + # Detect changed features files + comma_separated_changed_files = os.getenv('ALL_CHANGED_FILES') + changed_files = comma_separated_changed_files.split(',') + + #Check features file exists in services folder + services_path = os.getenv('SERVICES_PATH') + features_file_status = map_features_files_status(services_path, changed_files) + for file_path, exists in features_file_status.items(): + if not exists: + sys.stdout.write(f"::error title=Features file is missing::Service code was changed but no corresponding feature file found at {file_path}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/validate_features_files.py b/.github/scripts/validate_features_files.py new file mode 100644 index 0000000..aedf684 --- /dev/null +++ b/.github/scripts/validate_features_files.py @@ -0,0 +1,69 @@ +import json +import os + +import yaml +from jsonschema import validate, ValidationError +import sys + +FEATURES_FILE_NAME='features.yml' + +def load_yaml_file(file_path: str): + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except yaml.YAMLError as e: + print(f"Error parsing YAML file: {e}") + sys.exit(1) + except FileNotFoundError: + print(f"YAML file not found: {file_path}") + sys.exit(1) + +def load_json_file(file_path: str): + try: + with open(file_path, 'r') as file: + return json.load(file) + except json.JSONDecodeError as e: + print(f"Failed to parse JSON schema file: {e}") + sys.exit(1) + except FileNotFoundError: + print(f"Json schema file not found: {file_path}") + sys.exit(1) + +def validate_yaml_against_schema(yaml_data: str, json_schema: str, file_path: str) -> bool: + try: + validate(instance=yaml_data, schema=json_schema) + print(f"Successful validation of file: {file_path}") + return True + except ValidationError as e: + sys.stdout.write(f"::error file={file_path},title=Validation has failed at path {' -> '.join(str(x) for x in e.path)}::{e.message}") + return False + +def main(): + # Detect changed features files + comma_separated_changed_files = os.getenv('ALL_CHANGED_FILES') + if not comma_separated_changed_files: + print("Environment variable ALL_CHANGED_FILES is not set. Skipping validation.") + sys.exit(1) + + changed_files = comma_separated_changed_files.split(',') + changed_features_files = [path for path in changed_files if path.lower().endswith(FEATURES_FILE_NAME)] + print(f'Changed features files: {",".join(changed_features_files)}') + + if len(changed_features_files) == 0: + print('No feature file was changed. Skipping validation.') + sys.exit(0) + + features_schema_path = os.getenv('FEATURES_JSON_SCHEMA', 'features_schema.json') + features_schema = load_json_file(features_schema_path) + print(f'Features schema file was loaded: {features_schema_path}') + + # validate schemas + all_valid = True + for file_path in changed_features_files: + features_file = load_yaml_file(file_path) + if not validate_yaml_against_schema(features_file, features_schema, file_path): + all_valid = False + sys.exit(0 if all_valid else 1) + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pr-validate-features-files.yml b/.github/workflows/pr-validate-features-files.yml new file mode 100644 index 0000000..0fba995 --- /dev/null +++ b/.github/workflows/pr-validate-features-files.yml @@ -0,0 +1,65 @@ +name: Validate feature catalog files + +on: + workflow_call: + inputs: + aws_services_path: + description: 'Directory containing AWS service implementations in the localstack repository' + type: string + required: true + localstack_meta_ref: + description: 'Branch to checkout in the localstack/meta repository without "origin/" prefix' + type: string + required: false + +jobs: + validate-features-files: + name: Validate feature catalog files + runs-on: ubuntu-latest + + steps: + # Clone repository that's calling this reusable workflow + - name: Checkout current repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # Clone the localstack/meta repo to access required Python scripts and JSON schema files + - name: Checkout localstack/meta repository + uses: actions/checkout@v4 + with: + repository: 'localstack/meta' + sparse-checkout: | + .github/scripts + .github/features_schema.json + ref: ${{ inputs.localstack_meta_ref || 'main' }} + path: 'localstack-meta' + + - name: Fetch list of modified files in the PR + id: changed-files + run: | + ALL_CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r \ + --diff-filter=AM ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | paste -sd "," -) + echo "all_changed_files=$ALL_CHANGED_FILES" >> $GITHUB_OUTPUT + + - name: Set up Python + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python3 -m pip install pyyaml jsonschema + + - name: Check features files exist + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + SERVICES_PATH: "${{ inputs.aws_services_path }}" + run: python3 localstack-meta/.github/scripts/check_features_files_exist.py + + - name: Validate features files + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + FEATURES_JSON_SCHEMA: 'localstack-meta/.github/features_schema.json' + run: python3 localstack-meta/.github/scripts/validate_features_files.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7dee378 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.DS_Store + +.idea/ \ No newline at end of file