Asset loading test runner

README.md

@@ -147,14 +147,19 @@ Use this command to run tests on a package. Currently, there are two types of te
 
 These tests allow you to exercise any Ingest Node Pipelines defined by your packages.
 
-For details on how to configure pipeline test for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md).
+For details on how to configure and run pipeline tests for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md).
 
 #### System Tests
 
 These tests allow you to test a package's ability to ingest data end-to-end. 
 
-For details on how to configure amd run system tests, see the [HOWTO guide](docs/howto/system_testing.md).
+For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md).
 
+#### Asset Loading Tests
+
+These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected.
+
+For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md).
 
 ### `elastic-package version`
 

cmd/testrunner.go

@@ -6,6 +6,7 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -52,7 +53,6 @@ func setupTestCommand() *cobra.Command {
 
 	cmd.PersistentFlags().BoolP(cobraext.FailOnMissingFlagName, "m", false, cobraext.FailOnMissingFlagDescription)
 	cmd.PersistentFlags().BoolP(cobraext.GenerateTestResultFlagName, "g", false, cobraext.GenerateTestResultFlagDescription)
-	cmd.PersistentFlags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription)
 	cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription)
 	cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription)
 	cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription)
@@ -68,6 +68,10 @@ func setupTestCommand() *cobra.Command {
 			RunE:  action,
 		}
 
+		if runner.CanRunPerDataStream() {
+			testTypeCmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription)
+		}
+
 		cmd.AddCommand(testTypeCmd)
 	}
 
@@ -84,11 +88,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 			return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName)
 		}
 
-		dataStreams, err := cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName)
-		if err != nil {
-			return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName)
-		}
-
 		generateTestResult, err := cmd.Flags().GetBool(cobraext.GenerateTestResultFlagName)
 		if err != nil {
 			return cobraext.FlagParsingError(err, cobraext.GenerateTestResultFlagName)
@@ -112,16 +111,37 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 			return errors.Wrap(err, "locating package root failed")
 		}
 
-		testFolders, err := testrunner.FindTestFolders(packageRootPath, testType, dataStreams)
-		if err != nil {
-			return errors.Wrap(err, "unable to determine test folder paths")
-		}
+		var testFolders []testrunner.TestFolder
+		if runner.CanRunPerDataStream() {
+			var dataStreams []string
+			// We check for the existence of the data streams flag before trying to
+			// parse it because if the root test command is run instead of one of the
+			// subcommands of test, the data streams flag will not be defined.
+			if cmd.Flags().Lookup(cobraext.DataStreamsFlagName) != nil {
+				dataStreams, err = cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName)
+				if err != nil {
+					return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName)
+				}
+			}
+
+			testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType)
+			if err != nil {
+				return errors.Wrap(err, "unable to determine test folder paths")
+			}
 
-		if failOnMissing && len(testFolders) == 0 {
-			if len(dataStreams) > 0 {
-				return fmt.Errorf("no %s tests found for %s data stream(s)", testType, strings.Join(dataStreams, ","))
+			if failOnMissing && len(testFolders) == 0 {
+				if len(dataStreams) > 0 {
+					return fmt.Errorf("no %s tests found for %s data stream(s)", testType, strings.Join(dataStreams, ","))
+				}
+				return fmt.Errorf("no %s tests found", testType)
+			}
+		} else {
+			_, pkg := filepath.Split(packageRootPath)
+			testFolders = []testrunner.TestFolder{
+				{
+					Package: pkg,
+				},
 			}
-			return fmt.Errorf("no %s tests found", testType)
 		}
 
 		deferCleanup, err := cmd.Flags().GetDuration(cobraext.DeferCleanupFlagName)

docs/howto/asset_testing.md

@@ -0,0 +1,57 @@
+# HOWTO: Writing asset loading tests for a package
+
+## Introduction
+
+Elastic Packages define assets to be loaded into Elasticsearch and Kibana. Asset loading tests exercise installing a package to ensure that its assets are loaded into Elasticsearch and Kibana as expected.
+
+## Conceptual process
+
+Conceptually, running an asset load test involves the following steps:
+
+1. Build the package.
+1. Deploy Elasticsearch, Kibana, and the Package Registry (all part of the Elastic Stack). This step takes time so it should typically be done once as a pre-requisite to running asset loading tests on multiple packages.
+1. Install the package.
+1. Use various Kibana and Elasticsearch APIs to assert that the package's assets were loaded into Kibana and Elasticsearch as expected.
+1. Remove the package.
+
+## Defining an asset loading test
+
+As a package developer, you do not need to do any work to define an asset loading test for your package. All the necessary information is already present in the package's files.
+
+## Running an asset loading test
+
+First, you must build your package. This corresponds to step 1 as described in the [_Conceptual process_](#Conceptual-process) section.
+
+Navigate to the package's root folder (or any sub-folder under it) and run the following command.
+
+```
+elastic-package build
+```
+
+Next, you must deploy Elasticsearch, Kibana, and the Package Registry. This corresponds to step 2 as described in the [_Conceptual process_](#Conceptual-process) section.
+
+```
+elastic-package stack up -d
+```
+
+For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
+
+Next, you must set environment variables needed for further `elastic-package` commands.
+
+```
+$(elastic-package stack shellinit)
+```
+
+Next, you must invoke the asset loading test runner. This corresponds to steps 3 through 5 as described in the [_Conceptual process_](#Conceptual-process) section.
+
+Navigate to the package's root folder (or any sub-folder under it) and run the following command.
+
+```
+elastic-package test asset
+```
+
+Finally, when you are done running all asset loading tests, bring down the Elastic Stack. This corresponds to step 4 as described in the [_Conceptual process_](#Conceptual-process) section.
+
+```
+elastic-package stack down
+```

internal/kibana/client.go

@@ -53,6 +53,10 @@ func (c *Client) put(resourcePath string, body []byte) (int, []byte, error) {
 	return c.sendRequest(http.MethodPut, resourcePath, body)
 }
 
+func (c *Client) delete(resourcePath string) (int, []byte, error) {
+	return c.sendRequest(http.MethodDelete, resourcePath, nil)
+}
+
 func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []byte, error) {
 	reqBody := bytes.NewReader(body)
 	base, err := url.Parse(c.host)

internal/kibana/packages.go

@@ -0,0 +1,52 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package kibana
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/packages"
+)
+
+// InstallPackage installs the given package in Fleet.
+func (c *Client) InstallPackage(pkg packages.PackageManifest) ([]packages.Asset, error) {
+	path := fmt.Sprintf("%s/epm/packages/%s-%s", FleetAPI, pkg.Name, pkg.Version)
+	statusCode, respBody, err := c.post(path, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not install package")
+	}
+
+	return processResults("install", statusCode, respBody)
+}
+
+// RemovePackage removes the given package from Fleet.
+func (c *Client) RemovePackage(pkg packages.PackageManifest) ([]packages.Asset, error) {
+	path := fmt.Sprintf("%s/epm/packages/%s-%s", FleetAPI, pkg.Name, pkg.Version)
+	statusCode, respBody, err := c.delete(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not delete package")
+	}
+
+	return processResults("remove", statusCode, respBody)
+}
+
+func processResults(action string, statusCode int, respBody []byte) ([]packages.Asset, error) {
+	if statusCode != 200 {
+		return nil, fmt.Errorf("could not %s package; API status code = %d; response body = %s", action, statusCode, respBody)
+	}
+
+	var resp struct {
+		Assets []packages.Asset `json:"response"`
+	}
+
+	if err := json.Unmarshal(respBody, &resp); err != nil {
+		return nil, errors.Wrapf(err, "could not convert %s package (response) to JSON", action)
+	}
+
+	return resp.Assets, nil
+}

internal/packages/assets.go

@@ -0,0 +1,168 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package packages
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/multierror"
+)
+
+// AssetType represents the type of package asset.
+type AssetType string
+
+// Supported asset types.
+const (
+	AssetTypeElasticsearchIndexTemplate  AssetType = "index_template"
+	AssetTypeElasticsearchIngestPipeline AssetType = "ingest_pipeline"
+
+	AssetTypeKibanaSavedSearch   AssetType = "search"
+	AssetTypeKibanaVisualization AssetType = "visualization"
+	AssetTypeKibanaDashboard     AssetType = "dashboard"
+	AssetTypeKibanaMap           AssetType = "map"
+	AssetTypeKibanaLens          AssetType = "lens"
+)
+
+// Asset represents a package asset to be loaded into Kibana or Elasticsearch.
+type Asset struct {
+	ID         string    `json:"id"`
+	Type       AssetType `json:"type"`
+	DataStream string
+}
+
+// LoadPackageAssets parses the package contents and returns a list of assets defined by the package.
+func LoadPackageAssets(pkgRootPath string) ([]Asset, error) {
+	assets, err := loadKibanaAssets(pkgRootPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not load kibana assets")
+	}
+
+	a, err := loadElasticsearchAssets(pkgRootPath)
+	if err != nil {
+		return a, errors.Wrap(err, "could not load elasticsearch assets")
+	}
+	assets = append(assets, a...)
+
+	return assets, nil
+}
+
+func loadKibanaAssets(pkgRootPath string) ([]Asset, error) {
+	kibanaAssetsFolderPath := filepath.Join(pkgRootPath, "kibana")
+
+	var (
+		errs multierror.Error
+
+		assetTypes = []AssetType{
+			AssetTypeKibanaDashboard,
+			AssetTypeKibanaVisualization,
+			AssetTypeKibanaSavedSearch,
+			AssetTypeKibanaMap,
+			AssetTypeKibanaLens,
+		}
+
+		assets []Asset
+	)
+
+	for _, assetType := range assetTypes {
+		a, err := loadFileBasedAssets(kibanaAssetsFolderPath, assetType)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "could not load kibana %s assets", assetType))
+			continue
+		}
+
+		assets = append(assets, a...)
+	}
+
+	if len(errs) > 0 {
+		return nil, errs
+	}
+
+	return assets, nil
+}
+
+func loadElasticsearchAssets(pkgRootPath string) ([]Asset, error) {
+	packageManifestPath := filepath.Join(pkgRootPath, PackageManifestFile)
+	pkgManifest, err := ReadPackageManifest(packageManifestPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading package manifest file failed")
+	}
+
+	dataStreamManifestPaths, err := filepath.Glob(filepath.Join(pkgRootPath, "data_stream", "*", DataStreamManifestFile))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not read data stream manifest file paths")
+	}
+
+	var assets []Asset
+	for _, dsManifestPath := range dataStreamManifestPaths {
+		dsManifest, err := ReadDataStreamManifest(dsManifestPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "reading data stream manifest failed")
+		}
+
+		indexTemplateName := fmt.Sprintf("%s-%s.%s", dsManifest.Type, pkgManifest.Name, dsManifest.Name)
+		asset := Asset{
+			ID:         indexTemplateName,
+			Type:       AssetTypeElasticsearchIndexTemplate,
+			DataStream: dsManifest.Name,
+		}
+		assets = append(assets, asset)
+
+		if dsManifest.Type == dataStreamTypeLogs {
+			ingestPipelineName := dsManifest.GetPipelineNameOrDefault()
+			if ingestPipelineName == defaultPipelineName {
+				ingestPipelineName = fmt.Sprintf("%s-%s.%s-%s", dsManifest.Type, pkgManifest.Name, dsManifest.Name, pkgManifest.Version)
+			}
+			asset = Asset{
+				ID:         ingestPipelineName,
+				Type:       AssetTypeElasticsearchIngestPipeline,
+				DataStream: dsManifest.Name,
+			}
+			assets = append(assets, asset)
+		}
+	}
+
+	return assets, nil
+}
+
+func loadFileBasedAssets(kibanaAssetsFolderPath string, assetType AssetType) ([]Asset, error) {
+	assetsFolderPath := filepath.Join(kibanaAssetsFolderPath, string(assetType))
+	_, err := os.Stat(assetsFolderPath)
+	if err != nil && os.IsNotExist(err) {
+		// No assets folder defined; nothing to load
+		return nil, nil
+	}
+	if err != nil {
+		return nil, errors.Wrapf(err, "error finding kibana %s assets folder", assetType)
+	}
+
+	files, err := ioutil.ReadDir(assetsFolderPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not read %s files", assetType)
+	}
+
+	var assets []Asset
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+
+		name := f.Name()
+		id := strings.TrimSuffix(name, ".json")
+
+		asset := Asset{
+			ID:   id,
+			Type: assetType,
+		}
+		assets = append(assets, asset)
+	}
+
+	return assets, nil
+}