Add ability to skip tests (#221)

* Adding skipped to reporters

* Adding skipped to pipeline test runner

* Adding skipped to system test runner

* Adding skipped field to test result

* Add skip configuration for asset loading tests

* Refactoring: extracting result composer for reuse in multiple packages

* Use result composer in asset loading test runner

* Adding godoc

* Refactoring: extract skip config struct

* Adding license header + struct godoc

* Account for no test config

* Fix error message

* Adding skip section to test package

* Add warning logs for skipped tests

* Include test result

* Stringify link

* Using result

* Fixing format

* Make skipped optional

* Add skip reason and link

* Introducing SkippableConfig

* Introducing constructor for ResultComposer

* Temporarily commenting out skip so test runs again

* Making test fail with bad path

* Putting back skip section

Author: ycombinator

internal/testrunner/reporters/formats/human.go

@@ -36,6 +36,8 @@ func reportHumanFormat(results []testrunner.TestResult) (string, error) {
 			result = fmt.Sprintf("ERROR: %s", r.ErrorMsg)
 		} else if r.FailureMsg != "" {
 			result = fmt.Sprintf("FAIL: %s", r.FailureMsg)
+		} else if r.Skipped != nil {
+			result = r.Skipped.String()
 		} else {
 			result = "PASS"
 		}

internal/testrunner/reporters/formats/xunit.go

@@ -33,6 +33,7 @@ type testSuite struct {
 	NumTests    int    `xml:"tests,attr,omitempty"`
 	NumFailures int    `xml:"failures,attr,omitempty"`
 	NumErrors   int    `xml:"errors,attr,omitempty"`
+	NumSkipped  int    `xml:"skipped,attr,omitempty"`
 
 	Suites []testSuite `xml:"testsuite,omitempty"`
 	Cases  []testCase  `xml:"testcase,omitempty"`
@@ -42,15 +43,20 @@ type testCase struct {
 	ClassName     string  `xml:"classname,attr"`
 	TimeInSeconds float64 `xml:"time,attr"`
 
-	Error   string `xml:"error,omitempty"`
-	Failure string `xml:"failure,omitempty"`
+	Error   string   `xml:"error,omitempty"`
+	Failure string   `xml:"failure,omitempty"`
+	Skipped *skipped `xml:"skipped,omitempty"`
+}
+
+type skipped struct {
+	Message string `xml:"message,attr"`
 }
 
 func reportXUnitFormat(results []testrunner.TestResult) (string, error) {
 	// test type => package => data stream => test cases
 	tests := map[string]map[string]map[string][]testCase{}
 
-	var numTests, numFailures, numErrors int
+	var numTests, numFailures, numErrors, numSkipped int
 	for _, r := range results {
 		testType := string(r.TestType)
 		if _, exists := tests[testType]; !exists {
@@ -79,6 +85,10 @@ func reportXUnitFormat(results []testrunner.TestResult) (string, error) {
 			numErrors++
 		}
 
+		if r.Skipped != nil {
+			numSkipped++
+		}
+
 		name := fmt.Sprintf("%s test", r.TestType)
 		if r.Name != "" {
 			name += ": " + r.Name
@@ -91,6 +101,11 @@ func reportXUnitFormat(results []testrunner.TestResult) (string, error) {
 			Error:         r.ErrorMsg,
 			Failure:       failure,
 		}
+
+		if r.Skipped != nil {
+			c.Skipped = &skipped{r.Skipped.String()}
+		}
+
 		numTests++
 
 		tests[testType][r.Package][r.DataStream] = append(tests[testType][r.Package][r.DataStream], c)
@@ -107,6 +122,7 @@ func reportXUnitFormat(results []testrunner.TestResult) (string, error) {
 			NumTests:    numTests,
 			NumFailures: numFailures,
 			NumErrors:   numErrors,
+			NumSkipped:  numSkipped,
 
 			Cases: make([]testCase, 0),
 		}

internal/testrunner/runners/asset/runner.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
-	"time"
 
 	es "github.com/elastic/go-elasticsearch/v7"
 	"github.com/pkg/errors"
@@ -63,43 +62,39 @@ func (r runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, er
 }
 
 func (r *runner) run() ([]testrunner.TestResult, error) {
-	result := testrunner.TestResult{
+	result := testrunner.NewResultComposer(testrunner.TestResult{
 		TestType: TestType,
 		Package:  r.testFolder.Package,
-	}
+	})
 
-	startTime := time.Now()
-	resultsWith := func(tr testrunner.TestResult, err error) ([]testrunner.TestResult, error) {
-		tr.TimeElapsed = time.Now().Sub(startTime)
-		if err == nil {
-			return []testrunner.TestResult{tr}, nil
-		}
+	testConfig, err := newConfig(r.testFolder.Path)
+	if err != nil {
+		return result.WithError(errors.Wrap(err, "unable to load asset loading test config file"))
 
-		if tcf, ok := err.(testrunner.ErrTestCaseFailed); ok {
-			tr.FailureMsg = tcf.Reason
-			tr.FailureDetails = tcf.Details
-			return []testrunner.TestResult{tr}, nil
-		}
+	}
 
-		tr.ErrorMsg = err.Error()
-		return []testrunner.TestResult{tr}, err
+	if testConfig != nil && testConfig.Skip != nil {
+		logger.Warnf("skipping %s test for %s: %s (details: %s)",
+			TestType, r.testFolder.Package,
+			testConfig.Skip.Reason, testConfig.Skip.Link.String())
+		return result.WithSkip(testConfig.Skip)
 	}
 
 	pkgManifest, err := packages.ReadPackageManifest(filepath.Join(r.packageRootPath, packages.PackageManifestFile))
 	if err != nil {
-		return resultsWith(result, errors.Wrap(err, "reading package manifest failed"))
+		return result.WithError(errors.Wrap(err, "reading package manifest failed"))
 	}
 
 	// Install package
 	kib, err := kibana.NewClient()
 	if err != nil {
-		return resultsWith(result, errors.Wrap(err, "could not create kibana client"))
+		return result.WithError(errors.Wrap(err, "could not create kibana client"))
 	}
 
 	logger.Debug("installing package...")
 	actualAssets, err := kib.InstallPackage(*pkgManifest)
 	if err != nil {
-		return resultsWith(result, errors.Wrap(err, "could not install package"))
+		return result.WithError(errors.Wrap(err, "could not install package"))
 	}
 	r.removePackageHandler = func() error {
 		logger.Debug("removing package...")
@@ -111,26 +106,29 @@ func (r *runner) run() ([]testrunner.TestResult, error) {
 
 	expectedAssets, err := packages.LoadPackageAssets(r.packageRootPath)
 	if err != nil {
-		return resultsWith(result, errors.Wrap(err, "could not load expected package assets"))
+		return result.WithError(errors.Wrap(err, "could not load expected package assets"))
 	}
 
 	results := make([]testrunner.TestResult, 0, len(expectedAssets))
 	for _, e := range expectedAssets {
-		result := testrunner.TestResult{
-			Name:        fmt.Sprintf("%s %s is loaded", e.Type, e.ID),
-			Package:     pkgManifest.Name,
-			DataStream:  e.DataStream,
-			TestType:    TestType,
-			TimeElapsed: time.Now().Sub(startTime),
-		}
-
+		rc := testrunner.NewResultComposer(testrunner.TestResult{
+			Name:       fmt.Sprintf("%s %s is loaded", e.Type, e.ID),
+			Package:    pkgManifest.Name,
+			DataStream: e.DataStream,
+			TestType:   TestType,
+		})
+
+		var r []testrunner.TestResult
 		if !findActualAsset(actualAssets, e) {
-			result.FailureMsg = "could not find expected asset"
-			result.FailureDetails = fmt.Sprintf("could not find %s asset \"%s\". Assets loaded:\n%s", e.Type, e.ID, formatAssetsAsString(actualAssets))
+			r, _ = rc.WithError(testrunner.ErrTestCaseFailed{
+				Reason:  "could not find expected asset",
+				Details: fmt.Sprintf("could not find %s asset \"%s\". Assets loaded:\n%s", e.Type, e.ID, formatAssetsAsString(actualAssets)),
+			})
+		} else {
+			r, _ = rc.WithSuccess()
 		}
 
-		results = append(results, result)
-		startTime = time.Now()
+		results = append(results, r[0])
 	}
 
 	return results, nil

internal/testrunner/runners/asset/test_config.go

@@ -0,0 +1,47 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package asset
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/go-ucfg"
+	"github.com/elastic/go-ucfg/yaml"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/testrunner"
+)
+
+type testConfig struct {
+	testrunner.SkippableConfig `config:",inline"`
+}
+
+func newConfig(assetTestFolderPath string) (*testConfig, error) {
+	configFilePath := filepath.Join(assetTestFolderPath, "config.yml")
+
+	// Test configuration file is optional for asset loading tests. If it
+	// doesn't exist, we can return early.
+	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not load asset loading test configuration file: %s", configFilePath)
+	}
+
+	var c testConfig
+	cfg, err := yaml.NewConfig(data, ucfg.PathSep("."))
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to load asset loading test configuration file: %s", configFilePath)
+	}
+	if err := cfg.Unpack(&c); err != nil {
+		return nil, errors.Wrapf(err, "unable to unpack asset loading test configuration file: %s", configFilePath)
+	}
+
+	return &c, nil
+}

internal/testrunner/runners/pipeline/runner.go

@@ -107,6 +107,16 @@ func (r *runner) run() ([]testrunner.TestResult, error) {
 		}
 		tr.Name = tc.name
 
+		if tc.config.Skip != nil {
+			logger.Warnf("skipping %s test for %s/%s: %s (details: %s)",
+				TestType, r.options.TestFolder.Package, r.options.TestFolder.DataStream,
+				tc.config.Skip.Reason, tc.config.Skip.Link.String())
+
+			tr.Skipped = tc.config.Skip
+			results = append(results, tr)
+			continue
+		}
+
 		result, err := simulatePipelineProcessing(r.options.ESClient, entryPipeline, tc)
 		if err != nil {
 			err := errors.Wrap(err, "simulating pipeline processing failed")
@@ -170,6 +180,13 @@ func (r *runner) loadTestCaseFile(testCaseFile string) (*testCase, error) {
 		return nil, errors.Wrapf(err, "reading config for test case failed (testCasePath: %s)", testCasePath)
 	}
 
+	if config.Skip != nil {
+		return &testCase{
+			name:   testCaseFile,
+			config: &config,
+		}, nil
+	}
+
 	ext := filepath.Ext(testCaseFile)
 
 	var entries []json.RawMessage

internal/testrunner/runners/pipeline/test_config.go

@@ -12,11 +12,15 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/testrunner"
 )
 
 const configTestSuffix = "-config.json"
 
 type testConfig struct {
+	testrunner.SkippableConfig `config:",inline"`
+
 	Multiline     *multiline             `json:"multiline"`
 	Fields        map[string]interface{} `json:"fields"`
 	DynamicFields map[string]string      `json:"dynamic_fields"`