Skip to content

Commit 094ed90

Browse files
macnev2013macnev2013
authored
support for running all the test cases (#14)
1 parent aaeb7eb commit 094ed90

29 files changed

+788
-1199
lines changed

Diff for: ‎.circleci/config.yml

-88
This file was deleted.

Diff for: ‎.github/workflows/lint.yml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
on: [push, pull_request]
2+
3+
name: Linting
4+
jobs:
5+
lint:
6+
runs-on: ubuntu-latest
7+
steps:
8+
- uses: actions/checkout@v3
9+
- name: Set up Python 3.10.5
10+
uses: actions/setup-python@v4
11+
with:
12+
python-version: '3.10.5'
13+
cache: 'pip'
14+
- name: Install system dependencies
15+
run: |
16+
pip install --upgrade pip
17+
pip install -r requirements.txt
18+
- name: Run Linting
19+
run: |
20+
make lint

Diff for: ‎.github/workflows/main.yml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
on:
2+
schedule:
3+
- cron: '0 19 * * 6'
4+
workflow_dispatch:
5+
inputs:
6+
services:
7+
default: 'ls-community'
8+
type: string
9+
description: name of the service to execute tests for (e.g. "ls-community", "ls-pro", "ls-all", "s3,iam,ec2")
10+
localstack-image:
11+
required: false
12+
type: string
13+
default: 'localstack/localstack:latest'
14+
description: localstack docker image name to test against
15+
16+
name: Terraform Tests
17+
jobs:
18+
19+
prepare_list:
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v3
23+
with:
24+
submodules: 'true'
25+
- id: set-matrix
26+
run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT
27+
outputs:
28+
matrix: ${{ steps.set-matrix.outputs.matrix }}
29+
30+
test_service:
31+
needs: prepare_list
32+
strategy:
33+
max-parallel: 10
34+
fail-fast: false
35+
matrix:
36+
service: ${{ fromJson(needs.prepare_list.outputs.matrix) }}
37+
runs-on: ubuntu-latest
38+
39+
steps:
40+
41+
- uses: actions/checkout@v3
42+
with:
43+
submodules: 'true'
44+
45+
- uses: actions/setup-go@v3
46+
with:
47+
go-version: '1.18.x'
48+
cache: true
49+
cache-dependency-path: terraform-provider-aws/go.sum
50+
51+
- name: Set up Python 3.10.5
52+
uses: actions/setup-python@v4
53+
with:
54+
python-version: '3.10.5'
55+
cache: 'pip'
56+
57+
- name: Install system dependencies
58+
run: |
59+
pip install --upgrade pip
60+
pip install -r requirements.txt
61+
62+
- name: Patch Terraform Provider
63+
run: |
64+
cd terraform-provider-aws && go mod vendor
65+
cd ../
66+
python -m terraform_pytest.main patch
67+
68+
- name: Build ${{ matrix.service }} Binary
69+
run: |
70+
python -m terraform_pytest.main build -s ${{ matrix.service }}
71+
ls -la terraform-provider-aws/test-bin
72+
73+
- name: Run ${{ matrix.service }} Tests
74+
env:
75+
PYTEST_PARALLEL_CONFIG: "${{ matrix.service == 'ec2' && '-n 2' || '' }}"
76+
run: |
77+
python -m pytest $PYTEST_PARALLEL_CONFIG --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }}
78+
79+
- name: Publish ${{ matrix.service }} Test Results
80+
uses: EnricoMi/publish-unit-test-result-action@v2
81+
if: always()
82+
with:
83+
junit_files: target/reports/*.xml
84+
check_name: ${{ matrix.service }} Terraform Test Results

Diff for: ‎.gitignore

+8-9
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
*~
2-
*.swp
3-
*.log
4-
*.bak
5-
6-
.vscode
1+
.DS_Store
72
.idea
8-
*.iml
9-
10-
build/
3+
.venv
4+
.pytest_cache
5+
__pycache__
6+
target
7+
**/*.test
8+
report.xml
9+
volume

Diff for: ‎.gitmodules

-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
1-
[submodule "localstack"]
2-
path = localstack
3-
url = https://github.com/localstack/localstack.git
41
[submodule "terraform-provider-aws"]
52
path = terraform-provider-aws
63
url = https://github.com/hashicorp/terraform-provider-aws.git
7-
[submodule "moto"]
8-
path = moto
9-
url = [email protected]:localstack/moto

Diff for: ‎.pre-commit-config.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# See https://pre-commit.com for more information
2+
# See https://pre-commit.com/hooks.html for more hooks
3+
repos:
4+
- repo: https://github.com/psf/black
5+
rev: 22.3.0
6+
hooks:
7+
- id: black
8+
9+
- repo: https://github.com/pycqa/isort
10+
rev: 5.9.1
11+
hooks:
12+
- id: isort
13+
name: isort (python)
14+
- id: isort
15+
name: isort (cython)
16+
types: [cython]
17+
- id: isort
18+
name: isort (pyi)
19+
types: [pyi]

Diff for: ‎Makefile

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
VENV_BIN ?= python3 -m venv
4+
VENV_DIR ?= .venv
5+
PIP_CMD ?= pip3
6+
7+
ifeq ($(OS), Windows_NT)
8+
VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate
9+
else
10+
VENV_ACTIVATE = $(VENV_DIR)/bin/activate
11+
endif
12+
13+
usage: ## Show this help
14+
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s %s\n", $$1, $$2 }'
15+
16+
$(VENV_ACTIVATE):
17+
test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR)
18+
$(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux
19+
touch $(VENV_ACTIVATE)
20+
21+
VENV_RUN = . $(VENV_ACTIVATE)
22+
23+
venv: $(VENV_ACTIVATE) ## Create a new (empty) virtual environment
24+
25+
install: ## Install the package in editable mode
26+
$(VENV_RUN); $(PIP_CMD) install -r requirements.txt
27+
28+
init-precommit: ## install te pre-commit hook into your local git repository
29+
($(VENV_RUN); pre-commit install)
30+
31+
lint: ## Run linting
32+
@echo "Running black... "
33+
black --check .
34+
35+
format: ## Run formatting
36+
$(VENV_RUN); python -m isort .; python -m black .

Diff for: ‎README.md

+43-61
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,43 @@
1-
Terraform vs LocalStack
2-
=======================
3-
4-
This repository contains scripts and CI configurations to to run the Terraform Acceptance test suite of the AWS provider against LocalStack
5-
6-
## Utilities
7-
8-
Some utilities for local development:
9-
10-
* `bin/list-tests [--all]`: list the available tests by parsing the go test files.
11-
* `bin/install-aws-test` creates the binary for running the test suite (and installs it into `$HOME/.cache/localstack/aws.test`. requires go 1.16
12-
* `bin/run-tests [test]` run a specific test. this installs and runs localstack in a background process. add the flag `-t` to test against an already running localstack instance.
13-
14-
## Finding and running tests
15-
16-
After running `bin/install-aws-test`, use `bin/run-tests [OPTIONS...] [TESTS...]` to run individual tests or entire test suites.
17-
18-
Here are some examples:
19-
20-
* `bin/run-tests TestAccAWSAPIGatewayResource`
21-
* `bin/run-tests -t TestAccAWSAPIGatewayResource`: same as above, but does not start localstack
22-
* `bin/run-tests TestAccAWSAPIGateway`: runs all tests that match `TestAccAWSAPIGateway` (run `bin/list-tests TestAccAWSAPIGateway` to see which ones will be executed)
23-
* `bin/run-tests -e TestAccAWSAPIGatewayV2 TestAccAWSAPIGateway`: same as above, but excludes all tests that match `TestAccAWSAPIGatewayV2`.
24-
* `bin/run-tests -i localstack-tests.incl.txt`: runs all tests listed in the text file
25-
26-
You can use `bin/list-tests` with the same parameters to see which tests will be executed,
27-
or to find specific tests based on patterns.
28-
29-
For example:
30-
31-
```
32-
% bin/list-tests Queue
33-
TestAccAWSBatchJobQueue
34-
TestAccAWSGameliftGameSessionQueue
35-
TestAccAWSMediaConvertQueue
36-
TestAccAWSSQSQueue
37-
TestAccAWSSQSQueuePolicy
38-
TestAccDataSourceAwsBatchJobQueue
39-
TestAccDataSourceAwsSqsQueue
40-
```
41-
42-
or
43-
44-
```
45-
% bin/list-tests "Data.*Queue"
46-
TestAccDataSourceAwsBatchJobQueue
47-
TestAccDataSourceAwsSqsQueue
48-
```
49-
50-
## Generating the test reports
51-
52-
Test logs are aggregated into `build/tests/*.log`, the command `bin/create-report` will create junit-like xml reports.
53-
These can then be rendered into html using `bin/create-report-html`, which also creates a summary page in `build/report.html`.
54-
For rendering html, you need `junit2html`.
55-
56-
## Travis config
57-
58-
### Build cache
59-
60-
The Travis-CI worker caches the built `aws.test` binary across builds.
61-
The first build may therefore take a while.
1+
# Localstack Terraform Test Runner
2+
3+
This is a test runner for localstack and terraform. It will run a test cases from the hashicrop [terraform provider aws](https://github.com/hashicorp/terraform-provider-aws.git) against Localstack Instance.
4+
5+
Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics.
6+
7+
## Installation
8+
1. Clone the repository with submodules
9+
- `git clone [email protected]:localstack/localstack-terraform-test.git --recurse-submodules`
10+
- Make sure you have the latest version of the submodules after switching to a different branch using `git submodule update --init --recursive`
11+
2. Run `make venv` to create a virtual environment
12+
3. Run `make install` to install the dependencies
13+
14+
## How to run?
15+
1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws
16+
- **Note: This operation is not idempotent. Please apply the patch only once.**
17+
2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module
18+
3. Now you are ready to use `python -m pytest` commands to list and run test cases from golang
19+
20+
## How to run test cases?
21+
- To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/<service> --collect-only -q`
22+
- To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service/<service>/<test-file> -k <test-case-name> --ls-start` or `python -m pytest terraform-provider-aws/internal/service/<service>/<test-file>::<test-case-name> --ls-start`
23+
- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service/<service>/<test-file>::<test-case-name> --ls-start`
24+
25+
## Default environment variables for Terraform Tests
26+
- **TF_ACC**: `1`
27+
- **AWS_ACCESS_KEY_ID**: `test`
28+
- **AWS_SECRET_ACCESS_KEY**: `test`
29+
- **AWS_DEFAULT_REGION**: `us-west-1`
30+
- **AWS_ALTERNATE_ACCESS_KEY_ID**: `test`
31+
- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test`
32+
- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test`
33+
- **AWS_ALTERNATE_REGION**: `us-east-2`
34+
- **AWS_THIRD_REGION**: `eu-west-1`
35+
36+
## Environment variables for Localstack
37+
- **DEBUG**: `1`
38+
- **PROVIDER_OVERRIDE_S3**: `asf`
39+
- **FAIL_FAST**: `1`
40+
41+
## Options
42+
- `--ls-start`: Start localstack instance before running the test cases
43+
- `--ls-image`: Specify the localstack image to use, default is `localstack/localstack:latest`

0 commit comments

Comments
 (0)