diff --git a/README.md b/README.md index b09ba12fc..f51fe36d5 100644 --- a/README.md +++ b/README.md @@ -147,14 +147,19 @@ Use this command to run tests on a package. Currently, there are two types of te These tests allow you to exercise any Ingest Node Pipelines defined by your packages. -For details on how to configure pipeline test for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md). +For details on how to configure and run pipeline tests for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md). #### System Tests These tests allow you to test a package's ability to ingest data end-to-end. -For details on how to configure amd run system tests, see the [HOWTO guide](docs/howto/system_testing.md). +For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md). +#### Asset Loading Tests + +These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected. + +For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md). ### `elastic-package version` diff --git a/cmd/testrunner.go b/cmd/testrunner.go index a2812c038..c214a5691 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "path/filepath" "strings" "github.com/pkg/errors" @@ -52,7 +53,6 @@ func setupTestCommand() *cobra.Command { cmd.PersistentFlags().BoolP(cobraext.FailOnMissingFlagName, "m", false, cobraext.FailOnMissingFlagDescription) cmd.PersistentFlags().BoolP(cobraext.GenerateTestResultFlagName, "g", false, cobraext.GenerateTestResultFlagDescription) - cmd.PersistentFlags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) @@ -68,6 +68,10 @@ func setupTestCommand() *cobra.Command { RunE: action, } + if runner.CanRunPerDataStream() { + testTypeCmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription) + } + cmd.AddCommand(testTypeCmd) } @@ -84,11 +88,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName) } - dataStreams, err := cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) - } - generateTestResult, err := cmd.Flags().GetBool(cobraext.GenerateTestResultFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.GenerateTestResultFlagName) @@ -112,16 +111,37 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return errors.Wrap(err, "locating package root failed") } - testFolders, err := testrunner.FindTestFolders(packageRootPath, testType, dataStreams) - if err != nil { - return errors.Wrap(err, "unable to determine test folder paths") - } + var testFolders []testrunner.TestFolder + if runner.CanRunPerDataStream() { + var dataStreams []string + // We check for the existence of the data streams flag before trying to + // parse it because if the root test command is run instead of one of the + // subcommands of test, the data streams flag will not be defined. + if cmd.Flags().Lookup(cobraext.DataStreamsFlagName) != nil { + dataStreams, err = cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) + } + } + + testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType) + if err != nil { + return errors.Wrap(err, "unable to determine test folder paths") + } - if failOnMissing && len(testFolders) == 0 { - if len(dataStreams) > 0 { - return fmt.Errorf("no %s tests found for %s data stream(s)", testType, strings.Join(dataStreams, ",")) + if failOnMissing && len(testFolders) == 0 { + if len(dataStreams) > 0 { + return fmt.Errorf("no %s tests found for %s data stream(s)", testType, strings.Join(dataStreams, ",")) + } + return fmt.Errorf("no %s tests found", testType) + } + } else { + _, pkg := filepath.Split(packageRootPath) + testFolders = []testrunner.TestFolder{ + { + Package: pkg, + }, } - return fmt.Errorf("no %s tests found", testType) } deferCleanup, err := cmd.Flags().GetDuration(cobraext.DeferCleanupFlagName) diff --git a/docs/howto/asset_testing.md b/docs/howto/asset_testing.md new file mode 100644 index 000000000..2d39752c1 --- /dev/null +++ b/docs/howto/asset_testing.md @@ -0,0 +1,57 @@ +# HOWTO: Writing asset loading tests for a package + +## Introduction + +Elastic Packages define assets to be loaded into Elasticsearch and Kibana. Asset loading tests exercise installing a package to ensure that its assets are loaded into Elasticsearch and Kibana as expected. + +## Conceptual process + +Conceptually, running an asset load test involves the following steps: + +1. Build the package. +1. Deploy Elasticsearch, Kibana, and the Package Registry (all part of the Elastic Stack). This step takes time so it should typically be done once as a pre-requisite to running asset loading tests on multiple packages. +1. Install the package. +1. Use various Kibana and Elasticsearch APIs to assert that the package's assets were loaded into Kibana and Elasticsearch as expected. +1. Remove the package. + +## Defining an asset loading test + +As a package developer, you do not need to do any work to define an asset loading test for your package. All the necessary information is already present in the package's files. + +## Running an asset loading test + +First, you must build your package. This corresponds to step 1 as described in the [_Conceptual process_](#Conceptual-process) section. + +Navigate to the package's root folder (or any sub-folder under it) and run the following command. + +``` +elastic-package build +``` + +Next, you must deploy Elasticsearch, Kibana, and the Package Registry. This corresponds to step 2 as described in the [_Conceptual process_](#Conceptual-process) section. + +``` +elastic-package stack up -d +``` + +For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, you must set environment variables needed for further `elastic-package` commands. + +``` +$(elastic-package stack shellinit) +``` + +Next, you must invoke the asset loading test runner. This corresponds to steps 3 through 5 as described in the [_Conceptual process_](#Conceptual-process) section. + +Navigate to the package's root folder (or any sub-folder under it) and run the following command. + +``` +elastic-package test asset +``` + +Finally, when you are done running all asset loading tests, bring down the Elastic Stack. This corresponds to step 4 as described in the [_Conceptual process_](#Conceptual-process) section. + +``` +elastic-package stack down +``` diff --git a/internal/kibana/client.go b/internal/kibana/client.go index c85c650bc..32371db13 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -53,6 +53,10 @@ func (c *Client) put(resourcePath string, body []byte) (int, []byte, error) { return c.sendRequest(http.MethodPut, resourcePath, body) } +func (c *Client) delete(resourcePath string) (int, []byte, error) { + return c.sendRequest(http.MethodDelete, resourcePath, nil) +} + func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []byte, error) { reqBody := bytes.NewReader(body) base, err := url.Parse(c.host) diff --git a/internal/kibana/packages.go b/internal/kibana/packages.go new file mode 100644 index 000000000..7dde7a0aa --- /dev/null +++ b/internal/kibana/packages.go @@ -0,0 +1,52 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package kibana + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/packages" +) + +// InstallPackage installs the given package in Fleet. +func (c *Client) InstallPackage(pkg packages.PackageManifest) ([]packages.Asset, error) { + path := fmt.Sprintf("%s/epm/packages/%s-%s", FleetAPI, pkg.Name, pkg.Version) + statusCode, respBody, err := c.post(path, nil) + if err != nil { + return nil, errors.Wrap(err, "could not install package") + } + + return processResults("install", statusCode, respBody) +} + +// RemovePackage removes the given package from Fleet. +func (c *Client) RemovePackage(pkg packages.PackageManifest) ([]packages.Asset, error) { + path := fmt.Sprintf("%s/epm/packages/%s-%s", FleetAPI, pkg.Name, pkg.Version) + statusCode, respBody, err := c.delete(path) + if err != nil { + return nil, errors.Wrap(err, "could not delete package") + } + + return processResults("remove", statusCode, respBody) +} + +func processResults(action string, statusCode int, respBody []byte) ([]packages.Asset, error) { + if statusCode != 200 { + return nil, fmt.Errorf("could not %s package; API status code = %d; response body = %s", action, statusCode, respBody) + } + + var resp struct { + Assets []packages.Asset `json:"response"` + } + + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, errors.Wrapf(err, "could not convert %s package (response) to JSON", action) + } + + return resp.Assets, nil +} diff --git a/internal/packages/assets.go b/internal/packages/assets.go new file mode 100644 index 000000000..8cf054e75 --- /dev/null +++ b/internal/packages/assets.go @@ -0,0 +1,168 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package packages + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/multierror" +) + +// AssetType represents the type of package asset. +type AssetType string + +// Supported asset types. +const ( + AssetTypeElasticsearchIndexTemplate AssetType = "index_template" + AssetTypeElasticsearchIngestPipeline AssetType = "ingest_pipeline" + + AssetTypeKibanaSavedSearch AssetType = "search" + AssetTypeKibanaVisualization AssetType = "visualization" + AssetTypeKibanaDashboard AssetType = "dashboard" + AssetTypeKibanaMap AssetType = "map" + AssetTypeKibanaLens AssetType = "lens" +) + +// Asset represents a package asset to be loaded into Kibana or Elasticsearch. +type Asset struct { + ID string `json:"id"` + Type AssetType `json:"type"` + DataStream string +} + +// LoadPackageAssets parses the package contents and returns a list of assets defined by the package. +func LoadPackageAssets(pkgRootPath string) ([]Asset, error) { + assets, err := loadKibanaAssets(pkgRootPath) + if err != nil { + return nil, errors.Wrap(err, "could not load kibana assets") + } + + a, err := loadElasticsearchAssets(pkgRootPath) + if err != nil { + return a, errors.Wrap(err, "could not load elasticsearch assets") + } + assets = append(assets, a...) + + return assets, nil +} + +func loadKibanaAssets(pkgRootPath string) ([]Asset, error) { + kibanaAssetsFolderPath := filepath.Join(pkgRootPath, "kibana") + + var ( + errs multierror.Error + + assetTypes = []AssetType{ + AssetTypeKibanaDashboard, + AssetTypeKibanaVisualization, + AssetTypeKibanaSavedSearch, + AssetTypeKibanaMap, + AssetTypeKibanaLens, + } + + assets []Asset + ) + + for _, assetType := range assetTypes { + a, err := loadFileBasedAssets(kibanaAssetsFolderPath, assetType) + if err != nil { + errs = append(errs, errors.Wrapf(err, "could not load kibana %s assets", assetType)) + continue + } + + assets = append(assets, a...) + } + + if len(errs) > 0 { + return nil, errs + } + + return assets, nil +} + +func loadElasticsearchAssets(pkgRootPath string) ([]Asset, error) { + packageManifestPath := filepath.Join(pkgRootPath, PackageManifestFile) + pkgManifest, err := ReadPackageManifest(packageManifestPath) + if err != nil { + return nil, errors.Wrap(err, "reading package manifest file failed") + } + + dataStreamManifestPaths, err := filepath.Glob(filepath.Join(pkgRootPath, "data_stream", "*", DataStreamManifestFile)) + if err != nil { + return nil, errors.Wrap(err, "could not read data stream manifest file paths") + } + + var assets []Asset + for _, dsManifestPath := range dataStreamManifestPaths { + dsManifest, err := ReadDataStreamManifest(dsManifestPath) + if err != nil { + return nil, errors.Wrap(err, "reading data stream manifest failed") + } + + indexTemplateName := fmt.Sprintf("%s-%s.%s", dsManifest.Type, pkgManifest.Name, dsManifest.Name) + asset := Asset{ + ID: indexTemplateName, + Type: AssetTypeElasticsearchIndexTemplate, + DataStream: dsManifest.Name, + } + assets = append(assets, asset) + + if dsManifest.Type == dataStreamTypeLogs { + ingestPipelineName := dsManifest.GetPipelineNameOrDefault() + if ingestPipelineName == defaultPipelineName { + ingestPipelineName = fmt.Sprintf("%s-%s.%s-%s", dsManifest.Type, pkgManifest.Name, dsManifest.Name, pkgManifest.Version) + } + asset = Asset{ + ID: ingestPipelineName, + Type: AssetTypeElasticsearchIngestPipeline, + DataStream: dsManifest.Name, + } + assets = append(assets, asset) + } + } + + return assets, nil +} + +func loadFileBasedAssets(kibanaAssetsFolderPath string, assetType AssetType) ([]Asset, error) { + assetsFolderPath := filepath.Join(kibanaAssetsFolderPath, string(assetType)) + _, err := os.Stat(assetsFolderPath) + if err != nil && os.IsNotExist(err) { + // No assets folder defined; nothing to load + return nil, nil + } + if err != nil { + return nil, errors.Wrapf(err, "error finding kibana %s assets folder", assetType) + } + + files, err := ioutil.ReadDir(assetsFolderPath) + if err != nil { + return nil, errors.Wrapf(err, "could not read %s files", assetType) + } + + var assets []Asset + for _, f := range files { + if f.IsDir() { + continue + } + + name := f.Name() + id := strings.TrimSuffix(name, ".json") + + asset := Asset{ + ID: id, + Type: assetType, + } + assets = append(assets, asset) + } + + return assets, nil +} diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 3bbefea11..029da2287 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -20,6 +20,11 @@ const ( // DataStreamManifestFile is the name of the data stream's manifest file. DataStreamManifestFile = "manifest.yml" + + defaultPipelineName = "default" + + dataStreamTypeLogs = "logs" + dataStreamTypeMetrics = "metrics" ) // VarValue represents a variable value as defined in a package or data stream @@ -200,6 +205,15 @@ func ReadDataStreamManifest(path string) (*DataStreamManifest, error) { return &m, nil } +// GetPipelineNameOrDefault returns the name of the data stream's pipeline, if one is explicitly defined in the +// data stream manifest. If not, the default pipeline name is returned. +func (dsm *DataStreamManifest) GetPipelineNameOrDefault() string { + if dsm.Elasticsearch != nil && dsm.Elasticsearch.IngestPipeline != nil && dsm.Elasticsearch.IngestPipeline.Name != "" { + return dsm.Elasticsearch.IngestPipeline.Name + } + return defaultPipelineName +} + // FindInputByType returns the input for the provided type. func (pt *PolicyTemplate) FindInputByType(inputType string) *Input { for _, input := range pt.Inputs { @@ -223,5 +237,5 @@ func isDataStreamManifest(path string) (bool, error) { if err != nil { return false, errors.Wrapf(err, "reading package manifest failed (path: %s)", path) } - return m.Title != "" && (m.Type == "logs" || m.Type == "metrics"), nil + return m.Title != "" && (m.Type == dataStreamTypeLogs || m.Type == dataStreamTypeMetrics), nil } diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go new file mode 100644 index 000000000..50deac183 --- /dev/null +++ b/internal/testrunner/runners/asset/runner.go @@ -0,0 +1,156 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package asset + +import ( + "fmt" + "path/filepath" + "time" + + es "github.com/elastic/go-elasticsearch/v7" + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/testrunner" +) + +func init() { + testrunner.RegisterRunner(&runner{}) +} + +const ( + // TestType defining asset loading tests + TestType testrunner.TestType = "asset" +) + +type runner struct { + testFolder testrunner.TestFolder + packageRootPath string + esClient *es.Client + + // Execution order of following handlers is defined in runner.tearDown() method. + removePackageHandler func() error +} + +// Type returns the type of test that can be run by this test runner. +func (r *runner) Type() testrunner.TestType { + return TestType +} + +// String returns the name of the test runner. +func (r runner) String() string { + return "asset loading" +} + +// CanRunPerDataStream returns whether this test runner can run on individual +// data streams within the package. +func (r runner) CanRunPerDataStream() bool { + return false +} + +// Run runs the asset loading tests +func (r runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) { + r.testFolder = options.TestFolder + r.packageRootPath = options.PackageRootPath + r.esClient = options.ESClient + + return r.run() +} + +func (r *runner) run() ([]testrunner.TestResult, error) { + result := testrunner.TestResult{ + TestType: TestType, + Package: r.testFolder.Package, + } + + startTime := time.Now() + resultsWith := func(tr testrunner.TestResult, err error) ([]testrunner.TestResult, error) { + tr.TimeElapsed = time.Now().Sub(startTime) + if err == nil { + return []testrunner.TestResult{tr}, nil + } + + if tcf, ok := err.(testrunner.ErrTestCaseFailed); ok { + tr.FailureMsg = tcf.Reason + tr.FailureDetails = tcf.Details + return []testrunner.TestResult{tr}, nil + } + + tr.ErrorMsg = err.Error() + return []testrunner.TestResult{tr}, err + } + + pkgManifest, err := packages.ReadPackageManifest(filepath.Join(r.packageRootPath, packages.PackageManifestFile)) + if err != nil { + return resultsWith(result, errors.Wrap(err, "reading package manifest failed")) + } + + // Install package + kib, err := kibana.NewClient() + if err != nil { + return resultsWith(result, errors.Wrap(err, "could not create kibana client")) + } + + logger.Debug("installing package...") + actualAssets, err := kib.InstallPackage(*pkgManifest) + if err != nil { + return resultsWith(result, errors.Wrap(err, "could not install package")) + } + r.removePackageHandler = func() error { + logger.Debug("removing package...") + if _, err := kib.RemovePackage(*pkgManifest); err != nil { + return errors.Wrap(err, "error cleaning up package") + } + return nil + } + + expectedAssets, err := packages.LoadPackageAssets(r.packageRootPath) + if err != nil { + return resultsWith(result, errors.Wrap(err, "could not load expected package assets")) + } + + results := make([]testrunner.TestResult, 0, len(expectedAssets)) + for _, e := range expectedAssets { + result := testrunner.TestResult{ + Name: fmt.Sprintf("%s %s is loaded", e.Type, e.ID), + Package: pkgManifest.Name, + DataStream: e.DataStream, + TestType: TestType, + TimeElapsed: time.Now().Sub(startTime), + } + + if !findActualAsset(actualAssets, e) { + result.FailureMsg = "could not find expected asset" + result.FailureDetails = fmt.Sprintf("could not find expected asset with ID = %s and type = %s. Assets loaded = %v", e.ID, e.Type, actualAssets) + } + + results = append(results, result) + startTime = time.Now() + } + + return results, nil +} + +func (r *runner) TearDown() error { + if r.removePackageHandler != nil { + if err := r.removePackageHandler(); err != nil { + return err + } + } + + return nil +} + +func findActualAsset(actualAssets []packages.Asset, expectedAsset packages.Asset) bool { + for _, a := range actualAssets { + if a.Type == expectedAsset.Type && a.ID == expectedAsset.ID { + return true + } + } + + return false +} diff --git a/internal/testrunner/runners/pipeline/ingest_pipeline.go b/internal/testrunner/runners/pipeline/ingest_pipeline.go index 5e0963181..5f64e13f6 100644 --- a/internal/testrunner/runners/pipeline/ingest_pipeline.go +++ b/internal/testrunner/runners/pipeline/ingest_pipeline.go @@ -24,8 +24,6 @@ import ( "github.com/elastic/elastic-package/internal/packages" ) -const defaultPipelineName = "default" - var ingestPipelineTag = regexp.MustCompile("{{\\s*IngestPipeline.+}}") type pipelineResource struct { @@ -58,7 +56,7 @@ func installIngestPipelines(esClient *elasticsearch.Client, dataStreamPath strin nonce := time.Now().UnixNano() - mainPipeline := getWithPipelineNameWithNonce(getPipelineNameOrDefault(dataStreamManifest), nonce) + mainPipeline := getWithPipelineNameWithNonce(dataStreamManifest.GetPipelineNameOrDefault(), nonce) pipelines, err := loadIngestPipelineFiles(dataStreamPath, nonce) if err != nil { return "", nil, errors.Wrap(err, "loading ingest pipeline files failed") @@ -76,13 +74,6 @@ func installIngestPipelines(esClient *elasticsearch.Client, dataStreamPath strin return mainPipeline, jsonPipelines, nil } -func getPipelineNameOrDefault(dsm *packages.DataStreamManifest) string { - if dsm.Elasticsearch != nil && dsm.Elasticsearch.IngestPipeline != nil && dsm.Elasticsearch.IngestPipeline.Name != "" { - return dsm.Elasticsearch.IngestPipeline.Name - } - return defaultPipelineName -} - func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]pipelineResource, error) { elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline") fis, err := ioutil.ReadDir(elasticsearchPath) diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index f1c2c9274..5a909e898 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -48,11 +48,17 @@ func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, e return r.run() } -// ShutDown shuts down the pipeline test runner. +// TearDown shuts down the pipeline test runner. func (r *runner) TearDown() error { return nil } +// CanRunPerDataStream returns whether this test runner can run on individual +// data streams within the package. +func (r *runner) CanRunPerDataStream() bool { + return true +} + func (r *runner) run() ([]testrunner.TestResult, error) { testCaseFiles, err := r.listTestCaseFiles() if err != nil { diff --git a/internal/testrunner/runners/runners.go b/internal/testrunner/runners/runners.go index 4e3534d34..c1faaf31f 100644 --- a/internal/testrunner/runners/runners.go +++ b/internal/testrunner/runners/runners.go @@ -6,6 +6,7 @@ package runners import ( // Registered test runners + _ "github.com/elastic/elastic-package/internal/testrunner/runners/asset" _ "github.com/elastic/elastic-package/internal/testrunner/runners/pipeline" _ "github.com/elastic/elastic-package/internal/testrunner/runners/system" ) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 16b2fd8bc..5a1f15e88 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -51,17 +51,6 @@ type runner struct { wipeDataStreamHandler func() error } -type stackSettings struct { - elasticsearch struct { - host string - username string - password string - } - kibana struct { - host string - } -} - // Type returns the type of test that can be run by this test runner. func (r *runner) Type() testrunner.TestType { return TestType @@ -72,6 +61,12 @@ func (r *runner) String() string { return "system" } +// CanRunPerDataStream returns whether this test runner can run on individual +// data streams within the package. +func (r *runner) CanRunPerDataStream() bool { + return true +} + // Run runs the system tests defined under the given folder func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) { r.options = options diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index c1716581f..9ef57b05f 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -45,6 +45,8 @@ type TestRunner interface { // TearDown cleans up any test runner resources. It must be called // after the test runner has finished executing. TearDown() error + + CanRunPerDataStream() bool } var runners = map[TestType]TestRunner{} @@ -89,7 +91,7 @@ type TestFolder struct { } // FindTestFolders finds test folders for the given package and, optionally, test type and data streams -func FindTestFolders(packageRootPath string, testType TestType, dataStreams []string) ([]TestFolder, error) { +func FindTestFolders(packageRootPath string, dataStreams []string, testType TestType) ([]TestFolder, error) { // Expected folder structure: // / // data_stream/ @@ -170,7 +172,6 @@ func Run(testType TestType, options TestOptions) ([]TestResult, error) { errs = append(errs, err, tdErr) return nil, errors.Wrap(err, "could not complete test run and teardown test runner") } - return nil, errors.Wrap(err, "could not complete test run") } @@ -186,6 +187,8 @@ func TestRunners() map[TestType]TestRunner { return runners } +// findTestFoldersPaths can only be called for test runners that require tests to be defined +// at the data stream level. func findTestFolderPaths(packageRootPath, dataStreamGlob, testTypeGlob string) ([]string, error) { testFoldersGlob := filepath.Join(packageRootPath, "data_stream", dataStreamGlob, "_dev", "test", testTypeGlob) paths, err := filepath.Glob(testFoldersGlob) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 74e796b71..bba3c31f5 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -21,6 +21,7 @@ cleanup() { trap cleanup EXIT +OLDPWD=$PWD # Build/check packages for d in test/packages/*/; do ( @@ -28,6 +29,7 @@ for d in test/packages/*/; do elastic-package check -v ) done +cd - # Boot up the stack elastic-package stack up -d -v @@ -41,4 +43,5 @@ for d in test/packages/*/; do # defer-cleanup is set to a short period to verify that the option is available elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s ) +cd - done \ No newline at end of file diff --git a/test/packages/apache/manifest.yml b/test/packages/apache/manifest.yml index 6a97965b5..e2ba25cb6 100644 --- a/test/packages/apache/manifest.yml +++ b/test/packages/apache/manifest.yml @@ -1,7 +1,10 @@ format_version: 1.0.0 name: apache title: Apache -version: 0.1.4 +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 license: basic description: Apache Integration type: integration diff --git a/test/packages/aws/manifest.yml b/test/packages/aws/manifest.yml index 60f498760..f80a9ef69 100644 --- a/test/packages/aws/manifest.yml +++ b/test/packages/aws/manifest.yml @@ -1,7 +1,10 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.3.9 +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 license: basic description: AWS Integration type: integration diff --git a/test/packages/multiinput/manifest.yml b/test/packages/multiinput/manifest.yml index 4c293aef9..e24700ca1 100644 --- a/test/packages/multiinput/manifest.yml +++ b/test/packages/multiinput/manifest.yml @@ -1,7 +1,10 @@ format_version: 1.0.0 name: multiinput title: Multi-input test -version: 0.0.1 +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 description: Test for multiple input tests categories: ["network"] release: experimental diff --git a/test/packages/nginx/manifest.yml b/test/packages/nginx/manifest.yml index 1b023d4fc..fc0bf202b 100644 --- a/test/packages/nginx/manifest.yml +++ b/test/packages/nginx/manifest.yml @@ -1,7 +1,10 @@ format_version: 1.0.0 name: nginx title: Nginx -version: 0.2.4 +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 license: basic description: Nginx Integration type: integration