From c54eaf62627003c7edd3199943c508aa1ab26fc9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:26:11 +0100 Subject: [PATCH 1/7] Move CSAPI only tests to tests/csapi This allows them to run in parallel with federation tests. --- ONBOARDING.md | 14 +- .../account_change_password_pushers_test.go | 2 +- .../account_change_password_test.go | 2 +- tests/{ => csapi}/account_deactivate_test.go | 2 +- .../apidoc_device_management_test.go | 2 +- tests/{ => csapi}/apidoc_login_test.go | 2 +- tests/{ => csapi}/apidoc_presence_test.go | 2 +- .../apidoc_profile_avatar_url_test.go | 2 +- .../apidoc_profile_displayname_test.go | 2 +- tests/{ => csapi}/apidoc_register_test.go | 2 +- .../apidoc_request_encoding_test.go | 2 +- tests/{ => csapi}/apidoc_room_create_test.go | 2 +- tests/{ => csapi}/apidoc_room_state_test.go | 2 +- tests/{ => csapi}/apidoc_version_test.go | 2 +- tests/csapi/main_test.go | 127 ++++++++++++++++++ tests/{ => csapi}/rooms_state_test.go | 2 +- tests/{ => csapi}/sync_filter_test.go | 2 +- tests/{ => csapi}/user_query_keys_test.go | 2 +- 18 files changed, 153 insertions(+), 20 deletions(-) rename tests/{ => csapi}/account_change_password_pushers_test.go (99%) rename tests/{ => csapi}/account_change_password_test.go (99%) rename tests/{ => csapi}/account_deactivate_test.go (99%) rename tests/{ => csapi}/apidoc_device_management_test.go (99%) rename tests/{ => csapi}/apidoc_login_test.go (99%) rename tests/{ => csapi}/apidoc_presence_test.go (98%) rename tests/{ => csapi}/apidoc_profile_avatar_url_test.go (98%) rename tests/{ => csapi}/apidoc_profile_displayname_test.go (98%) rename tests/{ => csapi}/apidoc_register_test.go (99%) rename tests/{ => csapi}/apidoc_request_encoding_test.go (97%) rename tests/{ => csapi}/apidoc_room_create_test.go (99%) rename tests/{ => csapi}/apidoc_room_state_test.go (99%) rename tests/{ => csapi}/apidoc_version_test.go (97%) create mode 100644 tests/csapi/main_test.go rename tests/{ => csapi}/rooms_state_test.go (98%) rename tests/{ => csapi}/sync_filter_test.go (99%) rename tests/{ => csapi}/user_query_keys_test.go (98%) diff --git a/ONBOARDING.md b/ONBOARDING.md index a8028b21..d4d48240 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -103,6 +103,13 @@ Adding `// sytest: ...` means `sytest_coverage.go` will know the test is convert when run! Use `go run sytest_coverage.go -v` to see the exact string to use, as they may be different to the one produced by an actual sytest run due to parameterised tests. +### Where should I put new tests? + +If the test *only* has CS API calls, then put it in `/tests/csapi`. If the test involves both CS API and Federation, or just Federation, put it in `/tests`. +This is because of how parallelisation works currently. All federation tests MUST be in the same directory due to the use of shared resources (for example, +the local Complement server always binds to `:8448` which is a problem if 2 fed tests want to do that at the same time). This will be resolved in the future +by the use of `.well-known` but at present this is how things stand. + ### Should I always make a new blueprint for a test? Probably not. Blueprints are costly, and they should only be made if there is a strong case for plenty of reuse among tests. In the same way that we don't always add fixtures to sytest, we should be sparing with adding blueprints. @@ -140,6 +147,9 @@ t.Run("parallel", func(t *testing.T) { }) ``` +Tests in a directory will run in parallel with tests in other directories by default. You can disable this by invoking `go test -p 1` which will +force a parallelisation factor of 1 (no parallelisation). + ### How should I do comments in the test? Add long prose to the start of the function to outline what it is you're testing (and why if it is unclear). For example: @@ -177,10 +187,6 @@ Use one of `t.Skipf(...)` or `t.SkipNow()`. Error will fail the test but continue execution, where Fatal will fail the test and quit. Use Fatal when continuing to run the test will result in programming errors (e.g nil exceptions). -### Why do I get the error "Error response from daemon: Conflict. The container name "/complement_rooms_state_alice.hs1_1" is already in use by container "c2d1d90c6cff7b7de2678b56c702bd1ff76ca72b930e8f2ca32eef3f2514ff3b". You have to remove (or rename) that container to be able to reuse that name."? - -The Docker daemon has a lag time between removing containers and them actually being removed. This means you cannot remove a container called 'foo' and immediately recreate it as 'foo'. To get around this, you need to use a different name. This probably means the namespace you have given the deployment is used by another test. Try changing it to something else e.g `Deploy(t, "rooms_state_2", b.BlueprintAlice.Name)` - ### How do I run tests inside my IDE? For VSCode, add to `settings.json`: diff --git a/tests/account_change_password_pushers_test.go b/tests/csapi/account_change_password_pushers_test.go similarity index 99% rename from tests/account_change_password_pushers_test.go rename to tests/csapi/account_change_password_pushers_test.go index c4532f86..f73c294c 100644 --- a/tests/account_change_password_pushers_test.go +++ b/tests/csapi/account_change_password_pushers_test.go @@ -1,6 +1,6 @@ // +build !dendrite_blacklist -package tests +package csapi_tests import ( "testing" diff --git a/tests/account_change_password_test.go b/tests/csapi/account_change_password_test.go similarity index 99% rename from tests/account_change_password_test.go rename to tests/csapi/account_change_password_test.go index 988e53f3..5a02eb78 100644 --- a/tests/account_change_password_test.go +++ b/tests/csapi/account_change_password_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "io/ioutil" diff --git a/tests/account_deactivate_test.go b/tests/csapi/account_deactivate_test.go similarity index 99% rename from tests/account_deactivate_test.go rename to tests/csapi/account_deactivate_test.go index faf678b1..61afb412 100644 --- a/tests/account_deactivate_test.go +++ b/tests/csapi/account_deactivate_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "net/http" diff --git a/tests/apidoc_device_management_test.go b/tests/csapi/apidoc_device_management_test.go similarity index 99% rename from tests/apidoc_device_management_test.go rename to tests/csapi/apidoc_device_management_test.go index 21563b4f..afe6e186 100644 --- a/tests/apidoc_device_management_test.go +++ b/tests/csapi/apidoc_device_management_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_login_test.go b/tests/csapi/apidoc_login_test.go similarity index 99% rename from tests/apidoc_login_test.go rename to tests/csapi/apidoc_login_test.go index 0682685a..96dd1842 100644 --- a/tests/apidoc_login_test.go +++ b/tests/csapi/apidoc_login_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "encoding/json" diff --git a/tests/apidoc_presence_test.go b/tests/csapi/apidoc_presence_test.go similarity index 98% rename from tests/apidoc_presence_test.go rename to tests/csapi/apidoc_presence_test.go index 317bd234..ce02a19d 100644 --- a/tests/apidoc_presence_test.go +++ b/tests/csapi/apidoc_presence_test.go @@ -2,7 +2,7 @@ // Rationale for being included in Dendrite's blacklist: https://github.com/matrix-org/complement/pull/104#discussion_r617646624 -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_profile_avatar_url_test.go b/tests/csapi/apidoc_profile_avatar_url_test.go similarity index 98% rename from tests/apidoc_profile_avatar_url_test.go rename to tests/csapi/apidoc_profile_avatar_url_test.go index ddfda77d..850034df 100644 --- a/tests/apidoc_profile_avatar_url_test.go +++ b/tests/csapi/apidoc_profile_avatar_url_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_profile_displayname_test.go b/tests/csapi/apidoc_profile_displayname_test.go similarity index 98% rename from tests/apidoc_profile_displayname_test.go rename to tests/csapi/apidoc_profile_displayname_test.go index a1b3b066..aeec2efe 100644 --- a/tests/apidoc_profile_displayname_test.go +++ b/tests/csapi/apidoc_profile_displayname_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_register_test.go b/tests/csapi/apidoc_register_test.go similarity index 99% rename from tests/apidoc_register_test.go rename to tests/csapi/apidoc_register_test.go index f0588858..c25bb9f7 100644 --- a/tests/apidoc_register_test.go +++ b/tests/csapi/apidoc_register_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "encoding/json" diff --git a/tests/apidoc_request_encoding_test.go b/tests/csapi/apidoc_request_encoding_test.go similarity index 97% rename from tests/apidoc_request_encoding_test.go rename to tests/csapi/apidoc_request_encoding_test.go index f5f803aa..2651de65 100644 --- a/tests/apidoc_request_encoding_test.go +++ b/tests/csapi/apidoc_request_encoding_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_room_create_test.go b/tests/csapi/apidoc_room_create_test.go similarity index 99% rename from tests/apidoc_room_create_test.go rename to tests/csapi/apidoc_room_create_test.go index 2d99e41e..bd36f344 100644 --- a/tests/apidoc_room_create_test.go +++ b/tests/csapi/apidoc_room_create_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/apidoc_room_state_test.go b/tests/csapi/apidoc_room_state_test.go similarity index 99% rename from tests/apidoc_room_state_test.go rename to tests/csapi/apidoc_room_state_test.go index 6d4bb30a..1b6872e5 100644 --- a/tests/apidoc_room_state_test.go +++ b/tests/csapi/apidoc_room_state_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "net/url" diff --git a/tests/apidoc_version_test.go b/tests/csapi/apidoc_version_test.go similarity index 97% rename from tests/apidoc_version_test.go rename to tests/csapi/apidoc_version_test.go index 9c8646e9..31fcd53c 100644 --- a/tests/apidoc_version_test.go +++ b/tests/csapi/apidoc_version_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "fmt" diff --git a/tests/csapi/main_test.go b/tests/csapi/main_test.go new file mode 100644 index 00000000..70cc4808 --- /dev/null +++ b/tests/csapi/main_test.go @@ -0,0 +1,127 @@ +package csapi_tests + +import ( + "context" + "fmt" + "log" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/complement/internal/b" + "github.com/matrix-org/complement/internal/config" + "github.com/matrix-org/complement/internal/docker" + "github.com/matrix-org/complement/internal/federation" +) + +var namespaceCounter uint64 + +// persist the complement builder which is set when the tests start via TestMain +var complementBuilder *docker.Builder + +// TestMain is the main entry point for Complement. +// +// It will clean up any old containers/images/networks from the previous run, then run the tests, then clean up +// again. No blueprints are made at this point as they are lazily made on demand. +func TestMain(m *testing.M) { + cfg := config.NewConfigFromEnvVars() + cfg.PackageNamespace = "csapi" + log.Printf("config: %+v", cfg) + builder, err := docker.NewBuilder(cfg) + if err != nil { + fmt.Printf("Error: %s", err) + os.Exit(1) + } + complementBuilder = builder + // remove any old images/containers/networks in case we died horribly before + builder.Cleanup() + + if os.Getenv("COMPLEMENT_CA") == "true" { + log.Printf("Running with Complement CA") + // make sure CA certs are generated + _, _, err = federation.GetOrCreateCaCert() + if err != nil { + fmt.Printf("Error: %s", err) + os.Exit(1) + } + } + + // we use GMSL which uses logrus by default. We don't want those logs in our test output unless they are Serious. + logrus.SetLevel(logrus.ErrorLevel) + + exitCode := m.Run() + builder.Cleanup() + os.Exit(exitCode) +} + +// Deploy will deploy the given blueprint or terminate the test. +// It will construct the blueprint if it doesn't already exist in the docker image cache. +// This function is the main setup function for all tests as it provides a deployment with +// which tests can interact with. +func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment { + t.Helper() + timeStartBlueprint := time.Now() + if complementBuilder == nil { + t.Fatalf("complementBuilder not set, did you forget to call TestMain?") + } + if err := complementBuilder.ConstructBlueprintsIfNotExist([]b.Blueprint{blueprint}); err != nil { + t.Fatalf("Deploy: Failed to construct blueprint: %s", err) + } + namespace := fmt.Sprintf("%d", atomic.AddUint64(&namespaceCounter, 1)) + d, err := docker.NewDeployer(namespace, complementBuilder.Config) + if err != nil { + t.Fatalf("Deploy: NewDeployer returned error %s", err) + } + timeStartDeploy := time.Now() + dep, err := d.Deploy(context.Background(), blueprint.Name) + if err != nil { + t.Fatalf("Deploy: Deploy returned error %s", err) + } + t.Logf("Deploy times: %v blueprints, %v containers", timeStartDeploy.Sub(timeStartBlueprint), time.Since(timeStartDeploy)) + return dep +} + +type Waiter struct { + mu sync.Mutex + ch chan bool + closed bool +} + +// NewWaiter returns a generic struct which can be waited on until `Waiter.Finish` is called. +// A Waiter is similar to a `sync.WaitGroup` of size 1, but without the ability to underflow and +// with built-in timeouts. +func NewWaiter() *Waiter { + return &Waiter{ + ch: make(chan bool), + mu: sync.Mutex{}, + } +} + +// Wait blocks until Finish() is called or until the timeout is reached. +// If the timeout is reached, the test is failed. +func (w *Waiter) Wait(t *testing.T, timeout time.Duration) { + t.Helper() + select { + case <-w.ch: + return + case <-time.After(timeout): + t.Fatalf("Wait: timed out after %f seconds.", timeout.Seconds()) + } +} + +// Finish will cause all goroutines waiting via Wait to stop waiting and return. +// Once this function has been called, subsequent calls to Wait will return immediately. +// To begin waiting again, make a new Waiter. +func (w *Waiter) Finish() { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + w.closed = true + close(w.ch) +} diff --git a/tests/rooms_state_test.go b/tests/csapi/rooms_state_test.go similarity index 98% rename from tests/rooms_state_test.go rename to tests/csapi/rooms_state_test.go index 097d57b0..4579f5c3 100644 --- a/tests/rooms_state_test.go +++ b/tests/csapi/rooms_state_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "testing" diff --git a/tests/sync_filter_test.go b/tests/csapi/sync_filter_test.go similarity index 99% rename from tests/sync_filter_test.go rename to tests/csapi/sync_filter_test.go index 3615b690..45491298 100644 --- a/tests/sync_filter_test.go +++ b/tests/csapi/sync_filter_test.go @@ -1,4 +1,4 @@ -package tests +package csapi_tests import ( "encoding/json" diff --git a/tests/user_query_keys_test.go b/tests/csapi/user_query_keys_test.go similarity index 98% rename from tests/user_query_keys_test.go rename to tests/csapi/user_query_keys_test.go index 0a988161..0f55c057 100644 --- a/tests/user_query_keys_test.go +++ b/tests/csapi/user_query_keys_test.go @@ -2,7 +2,7 @@ // Rationale for being included in Synapse's blacklist: https://github.com/matrix-org/synapse/issues/10354 -package tests +package csapi_tests import ( "testing" From d955be3d19bd82424b9298ff50b2aae672c67d15 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:28:31 +0100 Subject: [PATCH 2/7] go test ./... --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e9ea9297..cb17428c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Complement is a black box integration testing framework for Matrix homeservers. You need to have Go and Docker installed, as well as `libolm3` and `libolm-dev`. Then: ``` -$ COMPLEMENT_BASE_IMAGE=some-matrix/homeserver-impl COMPLEMENT_BASE_IMAGE_ARGS='-foo bar -baz 1' go test -v ./tests +$ COMPLEMENT_BASE_IMAGE=some-matrix/homeserver-impl COMPLEMENT_BASE_IMAGE_ARGS='-foo bar -baz 1' go test -v ./tests/... ``` You can install `libolm3` on Debian using something like: @@ -33,7 +33,7 @@ You can either use your own image, or one of the ones supplied in the [dockerfil A full list of config options can be found [in the config file](./internal/config/config.go). All normal Go test config options will work, so to just run 1 named test and include a timeout for the test run: ``` -$ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -timeout 30s -run '^(TestOutboundFederationSend)$' -v ./tests +$ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -timeout 30s -run '^(TestOutboundFederationSend)$' -v ./tests/... ``` ### Running against Dendrite @@ -43,7 +43,7 @@ For instance, for Dendrite: # build a docker image for Dendrite... $ (cd dockerfiles && docker build -t complement-dendrite -f Dendrite.Dockerfile .) # ...and test it -$ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v ./tests +$ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v ./tests/... ``` ### Running against Synapse @@ -61,7 +61,7 @@ To run Complement against a specific release of Synapse, set the ```sh docker build -t complement-synapse:v1.36.0 -f dockerfiles/Synapse.Dockerfile --build-arg=SYNAPSE_VERSION=v1.36.0 dockerfiles -COMPLEMENT_BASE_IMAGE=complement-synapse:v1.36.0 go test ./tests +COMPLEMENT_BASE_IMAGE=complement-synapse:v1.36.0 go test ./tests/... ``` ### Image requirements @@ -95,7 +95,7 @@ being picked up by `go test`. For example, `apidoc_presence_test.go` has: ``` and all Dendrite tests run with `-tags="dendrite_blacklist"` to cause this file to be skipped. You can run tests with build tags like this: ``` -COMPLEMENT_BASE_IMAGE=complement-synapse:latest go test -v -tags="synapse_blacklist,msc2403" ./tests +COMPLEMENT_BASE_IMAGE=complement-synapse:latest go test -v -tags="synapse_blacklist,msc2403" ./tests/... ``` This runs Complement with a Synapse HS and ignores tests which Synapse doesn't implement, and includes tests for MSC2403. From 200c4688e242810c0b07d13441e5a218ae45053b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:32:31 +0100 Subject: [PATCH 3/7] Update github actions --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fef9ecbd..6a452e82 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,6 @@ jobs: steps: - uses: actions/checkout@v2 - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/ - - run: go test -v -tags "${{ matrix.tags }}" ./tests + - run: go test -v -tags "${{ matrix.tags }}" ./tests/... env: COMPLEMENT_BASE_IMAGE: homeserver From 8bcccb725f22fef75f39896baaad1802b5c89103 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:40:29 +0100 Subject: [PATCH 4/7] Track number of cpu cores on GH actions --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6a452e82..4977e520 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,6 +40,8 @@ jobs: - /var/run/docker.sock:/var/run/docker.sock steps: + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 - uses: actions/checkout@v2 - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/ - run: go test -v -tags "${{ matrix.tags }}" ./tests/... From 80d3fd75281fadb5c80bc640a508b8123adf284a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:42:20 +0100 Subject: [PATCH 5/7] Linting --- tests/csapi/main_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/csapi/main_test.go b/tests/csapi/main_test.go index 70cc4808..829108c3 100644 --- a/tests/csapi/main_test.go +++ b/tests/csapi/main_test.go @@ -85,6 +85,7 @@ func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment { return dep } +// nolint:unused type Waiter struct { mu sync.Mutex ch chan bool @@ -94,6 +95,7 @@ type Waiter struct { // NewWaiter returns a generic struct which can be waited on until `Waiter.Finish` is called. // A Waiter is similar to a `sync.WaitGroup` of size 1, but without the ability to underflow and // with built-in timeouts. +// nolint:unused func NewWaiter() *Waiter { return &Waiter{ ch: make(chan bool), From 1f6d7063b78ec52b6ef2369bab400aacf1060a71 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:47:04 +0100 Subject: [PATCH 6/7] Force 2 parallel executions --- .github/workflows/ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4977e520..5a70aaaa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,10 +40,9 @@ jobs: - /var/run/docker.sock:/var/run/docker.sock steps: - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + - run: lscpu - uses: actions/checkout@v2 - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/ - - run: go test -v -tags "${{ matrix.tags }}" ./tests/... + - run: go test -p 2 -v -tags "${{ matrix.tags }}" ./tests/... env: COMPLEMENT_BASE_IMAGE: homeserver From 19f9a229c13a9dfe1e0cee4266b2ee863f3feac1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 22 Jul 2021 12:56:16 +0100 Subject: [PATCH 7/7] No more lscpu --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a70aaaa..a341eb05 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,6 @@ jobs: - /var/run/docker.sock:/var/run/docker.sock steps: - - run: lscpu - uses: actions/checkout@v2 - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/ - run: go test -p 2 -v -tags "${{ matrix.tags }}" ./tests/...