Creating github webhook secrets per pipeline (#283)

* Secrets are now prefixed for pipelines

* Creating github webhook secrets per pipeline

* Setting header before parsing

* Added missing value

* Added migrating from the old format to the new

* Changed the return value

* Closing the request body

* Fixed

* Saving the pipeline

* Saving the new pipeline

* Fixed the damn thing.

* fix to reset only on unstaged

* Fixed nodejs build

* Fixed ruby and the build order

* Fixed the test

* Fixed the ruby test

Author: Skarlso

gaia.go

@@ -155,6 +155,13 @@ const (
 
 	// StartReasonScheduled label for pipelines which were triggered automated process, i.e. cron job.
 	StartReasonScheduled = "scheduled"
+
+	// SecretNamePrefix defines the prefix for github secrets for pipelines.
+	SecretNamePrefix = "GITHUB_WEBHOOK_SECRET_"
+
+	// LegacySecretName is the old name for a secret that has been created by previous versions.
+	// Deprecated
+	LegacySecretName = "GITHUB_WEBHOOK_SECRET"
 )
 
 // JwtExpiry is the default JWT expiry.

providers/pipelines/hook.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/labstack/echo"
@@ -61,7 +62,7 @@ func verifySignature(secret []byte, signature string, body []byte) bool {
 	return hmac.Equal(expected, actual)
 }
 
-func parse(secret []byte, req *http.Request) (Hook, error) {
+func checkHeaders(req *http.Request) (Hook, error) {
 	h := Hook{}
 
 	if h.Signature = req.Header.Get("x-hub-signature"); len(h.Signature) == 0 {
@@ -82,19 +83,7 @@ func parse(secret []byte, req *http.Request) (Hook, error) {
 	if h.ID = req.Header.Get("x-github-delivery"); len(h.ID) == 0 {
 		return Hook{}, errors.New("no event id")
 	}
-
-	body, err := ioutil.ReadAll(req.Body)
-
-	if err != nil {
-		return Hook{}, err
-	}
-
-	if !verifySignature(secret, h.Signature, body) {
-		return Hook{}, errors.New("Invalid signature")
-	}
-
-	h.Payload = body
-	return h, err
+	return h, nil
 }
 
 // GitWebHook handles callbacks from GitHub's webhook system.
@@ -109,13 +98,11 @@ func (pp *PipelineProvider) GitWebHook(c echo.Context) error {
 		return c.String(http.StatusInternalServerError, "unable to open vault: "+err.Error())
 	}
 
-	secret, err := vault.Get("GITHUB_WEBHOOK_SECRET")
-	if err != nil {
-		return c.String(http.StatusBadRequest, err.Error())
-	}
+	req := c.Request()
+	req.Header.Set("Content-type", "application/json")
+	defer req.Body.Close()
 
-	h, err := parse(secret, c.Request())
-	c.Request().Header.Set("Content-type", "application/json")
+	h, err := checkHeaders(req)
 	if err != nil {
 		return c.String(http.StatusBadRequest, err.Error())
 	}
@@ -124,10 +111,14 @@ func (pp *PipelineProvider) GitWebHook(c echo.Context) error {
 	}
 
 	p := Payload{}
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return c.String(http.StatusBadRequest, err.Error())
+	}
+	h.Payload = body
 	if err := json.Unmarshal(h.Payload, &p); err != nil {
 		return c.String(http.StatusBadRequest, "error in unmarshalling json payload")
 	}
-
 	var foundPipeline *gaia.Pipeline
 	for _, pipe := range pipeline.GlobalActivePipelines.GetAll() {
 		if pipe.Repo.URL == p.Repo.GitURL || pipe.Repo.URL == p.Repo.HTMLURL || pipe.Repo.URL == p.Repo.SSHURL {
@@ -138,10 +129,35 @@ func (pp *PipelineProvider) GitWebHook(c echo.Context) error {
 	if foundPipeline == nil {
 		return c.String(http.StatusInternalServerError, "pipeline not found")
 	}
+	id := strconv.Itoa(foundPipeline.ID)
+	secret, err := vault.Get(gaia.SecretNamePrefix + id)
+	migrate := false
+	if err != nil {
+		// Backwards compatibility, check if there is a secret using the old name.
+		secret, err = vault.Get(gaia.LegacySecretName)
+		if err != nil {
+			return c.String(http.StatusBadRequest, err.Error())
+		}
+		migrate = true
+		err = nil
+	}
+	if !verifySignature(secret, h.Signature, h.Payload) {
+		return c.String(http.StatusBadRequest, "invalid signature")
+	}
+
+	if migrate {
+		// Migrate the secret to a new value using the new format after verification of the signature succeeded.
+		// We don't want to migrate an incorrect secret.
+		vault.Add(gaia.SecretNamePrefix+id, secret)
+		if err := vault.SaveSecrets(); err != nil {
+			return c.String(http.StatusBadRequest, err.Error())
+		}
+	}
+
 	uniqueFolder, err := pipelinehelper.GetLocalDestinationForPipeline(*foundPipeline)
 	if err != nil {
 		gaia.Cfg.Logger.Error("Pipeline type invalid", "type", foundPipeline.Type)
-		return err
+		return c.String(http.StatusInternalServerError, "pipeline type invalid")
 	}
 	foundPipeline.Repo.LocalDest = uniqueFolder
 	err = pp.deps.PipelineService.UpdateRepository(foundPipeline)

providers/pipelines/pipeline_test.go

@@ -641,7 +641,7 @@ func TestHookReceive(t *testing.T) {
 	})
 	m := new(MockVaultStorer)
 	v, _ := services.VaultService(m)
-	v.Add("GITHUB_WEBHOOK_SECRET", []byte("superawesomesecretgithubpassword"))
+	v.Add("GITHUB_WEBHOOK_SECRET_1", []byte("superawesomesecretgithubpassword"))
 	defer func() {
 		services.MockVaultService(nil)
 	}()

workers/pipeline/build_ruby.go

@@ -29,7 +29,8 @@ const gemInitFile = "gaia.rb"
 
 // BuildPipelineRuby is the real implementation of BuildPipeline for Ruby
 type BuildPipelineRuby struct {
-	Type gaia.PipelineType
+	Type        gaia.PipelineType
+	GemfileName string
 }
 
 // PrepareEnvironment prepares the environment before we start the build process.
@@ -131,22 +132,31 @@ func (b *BuildPipelineRuby) ExecuteBuild(p *gaia.CreatePipeline) error {
 	}
 
 	// Search for resulting gem file.
-	gemfile, err := filterPathContentBySuffix(localDest, ".gem")
+	gemfile, err := findGemFileByGlob(localDest, uuid+"*.gem")
 	if err != nil {
 		gaia.Cfg.Logger.Error("cannot find final gem file after build", "path", p.Pipeline.Repo.LocalDest)
 		return err
 	}
 
-	// if we found more or less than one gem file then we have a problem.
+	// fallback to the old method because we don't have a uuid based gem file.
+	if gemfile == nil {
+		gemfile, err = findGemFileByGlob(localDest, "*.gem")
+		if err != nil {
+			gaia.Cfg.Logger.Error("cannot find final gem file after build", "path", p.Pipeline.Repo.LocalDest)
+			return err
+		}
+	}
+
+	// if we found more or less than one gem file for the given uuid then we have a problem.
 	if len(gemfile) != 1 {
-		gaia.Cfg.Logger.Debug("cannot find gem file in cloned repo", "foundGemFiles", len(gemfile), "gems", gemfile)
+		gaia.Cfg.Logger.Debug("cannot find gem file in cloned repo", "gems", gemfile)
 		return errors.New("cannot find gem file in cloned repo")
 	}
 
 	// Build has been finished. Set execution path to the build result archive.
 	// This will be used during pipeline verification phase which will happen after this step.
 	p.Pipeline.ExecPath = gemfile[0]
-
+	b.GemfileName = gemfile[0]
 	return nil
 }
 
@@ -170,18 +180,19 @@ func filterPathContentBySuffix(path, suffix string) ([]string, error) {
 	return filteredFiles, nil
 }
 
+// findGemFileByGlob reads the whole directory given by path and
+// returns all files with full path which have the same glob like provided.
+func findGemFileByGlob(path, glob string) ([]string, error) {
+	fullPath := filepath.Join(path, glob)
+	return filepath.Glob(fullPath)
+}
+
 // CopyBinary copies the final compiled binary to the
 // destination folder.
 func (b *BuildPipelineRuby) CopyBinary(p *gaia.CreatePipeline) error {
-	// Search for resulting gem file.
-	gemfile, err := filterPathContentBySuffix(p.Pipeline.Repo.LocalDest, ".gem")
-	if err != nil {
-		gaia.Cfg.Logger.Error("cannot find final gem file during copy", "path", p.Pipeline.Repo.LocalDest)
-		return err
-	}
-
 	// Define src and destination
-	src := gemfile[0]
+	src := b.GemfileName
+	gaia.Cfg.Logger.Debug("Copying over ruby gem file", "gem", src)
 	dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type))
 
 	// Copy binary

workers/pipeline/build_ruby_test.go

@@ -150,6 +150,7 @@ func TestCopyBinaryRuby(t *testing.T) {
 	p.Pipeline.Type = gaia.PTypeRuby
 	p.Pipeline.Repo = &gaia.GitRepo{LocalDest: tmp}
 	src := filepath.Join(tmp, "test.gem")
+	b.GemfileName = src
 	dst := pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)
 	f, _ := os.Create(src)
 	defer f.Close()
@@ -184,6 +185,7 @@ func TestCopyBinarySrcDoesNotExistRuby(t *testing.T) {
 	p.Pipeline.Name = "main"
 	p.Pipeline.Type = gaia.PTypeRuby
 	p.Pipeline.Repo = &gaia.GitRepo{LocalDest: "/noneexistent"}
+	b.GemfileName = "/noneexistent"
 	err := b.CopyBinary(p)
 	if err == nil {
 		t.Fatal("error was expected when copying binary but none occurred ")

workers/pipeline/create_pipeline.go

@@ -3,6 +3,7 @@ package pipeline
 import (
 	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 	"unicode"
 
@@ -158,7 +159,8 @@ func (s *GaiaPipelineService) CreatePipeline(p *gaia.CreatePipeline) {
 
 	if !gaia.Cfg.Poll && len(gitToken) > 0 {
 		// if there is a githubtoken provided, that means that a webhook was requested to be added.
-		err = createGithubWebhook(gitToken, p.Pipeline.Repo, nil)
+		id := strconv.Itoa(p.Pipeline.ID)
+		err = createGithubWebhook(gitToken, p.Pipeline.Repo, id, nil)
 		if err != nil {
 			gaia.Cfg.Logger.Error("error while creating webhook for repository", "error", err.Error())
 			return
@@ -192,7 +194,7 @@ func ValidatePipelineName(pName string) error {
 
 		// Check if pipeline name is already in use.
 		for _, activePipeline := range GlobalActivePipelines.GetAll() {
-			if strings.ToLower(s) == strings.ToLower(activePipeline.Name) {
+			if strings.EqualFold(s, activePipeline.Name) {
 				return errPipelineNameInUse
 			}
 		}

workers/pipeline/create_pipeline_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"io/ioutil"
+	"log"
 	"strings"
 	"testing"
 
@@ -134,13 +135,15 @@ func TestCreatePipeline(t *testing.T) {
 	defer func() { services.MockStorageService(nil) }()
 	cp := new(gaia.CreatePipeline)
 	cp.Pipeline.Name = "test"
+	cp.Pipeline.ID = 1
 	cp.Pipeline.Type = gaia.PTypeGolang
 	cp.Pipeline.Repo = &gaia.GitRepo{URL: "https://github.com/gaia-pipeline/pipeline-test"}
 	pipelineService := NewGaiaPipelineService(Dependencies{
 		Scheduler: &mockScheduleService{},
 	})
 	pipelineService.CreatePipeline(cp)
 	if cp.StatusType != gaia.CreatePipelineSuccess {
+		log.Println("Output: ", cp.Output)
 		t.Fatal("pipeline status was not success. was: ", cp.StatusType)
 	}
 }

workers/pipeline/git.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	gohttp "net/http"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -95,6 +96,10 @@ func GitLSRemote(repo *gaia.GitRepo) error {
 // UpdateRepository takes a git type repository and updates
 // it by pulling in new code if it's available.
 func (s *GaiaPipelineService) UpdateRepository(pipe *gaia.Pipeline) error {
+	gaia.Cfg.Logger.Debug("updating repository for pipeline type", "type", pipe.Type)
+	if pipe.Type == gaia.PTypeNodeJS {
+		pipe.Repo.LocalDest = filepath.Join(pipe.Repo.LocalDest, nodeJSInternalCloneFolder)
+	}
 	r, err := git.PlainOpen(pipe.Repo.LocalDest)
 	if err != nil {
 		// We don't stop gaia working because of an automated update failed.
@@ -110,7 +115,9 @@ func (s *GaiaPipelineService) UpdateRepository(pipe *gaia.Pipeline) error {
 		gaia.Cfg.Logger.Error("error getting auth info while doing a pull request: ", "error", err.Error())
 		return err
 	}
+
 	tree, _ := r.Worktree()
+
 	o := &git.PullOptions{
 		ReferenceName: plumbing.ReferenceName(pipe.Repo.SelectedBranch),
 		SingleBranch:  true,
@@ -126,12 +133,20 @@ func (s *GaiaPipelineService) UpdateRepository(pipe *gaia.Pipeline) error {
 				return err
 			}
 			o.Auth = auth
-			err = tree.Pull(o)
-			if err != nil {
+			if err := tree.Pull(o); err != nil {
 				return err
 			}
 		} else if strings.Contains(err.Error(), "worktree contains unstaged changes") {
-			// ignore this error, the pull overwrote everything anyways.
+			gaia.Cfg.Logger.Error("worktree contains unstaged changes, resetting", "error", err.Error())
+			// Clean the worktree. Because of various builds, it can happen that the local folder if polluted.
+			// For example go build tends to edit the go.mod file.
+			if err := tree.Reset(&git.ResetOptions{
+				Mode: git.HardReset,
+			}); err != nil {
+				gaia.Cfg.Logger.Error("failed to reset worktree", "error", err.Error())
+				return err
+			}
+			// Success, move on.
 			err = nil
 		} else {
 			// It's also an error if the repo is already up to date so we just move on.
@@ -142,14 +157,19 @@ func (s *GaiaPipelineService) UpdateRepository(pipe *gaia.Pipeline) error {
 
 	gaia.Cfg.Logger.Debug("updating pipeline: ", "message", pipe.Name)
 	b := newBuildPipeline(pipe.Type)
-	createPipeline := &gaia.CreatePipeline{
-		Pipeline: gaia.Pipeline{
-			Repo: &gaia.GitRepo{},
-		},
+	createPipeline := &gaia.CreatePipeline{Pipeline: *pipe}
+	if err := b.ExecuteBuild(createPipeline); err != nil {
+		gaia.Cfg.Logger.Error("error while executing the build", "error", err.Error())
+		return err
+	}
+	if err := b.SavePipeline(&createPipeline.Pipeline); err != nil {
+		gaia.Cfg.Logger.Error("failed to save pipeline", "error", err.Error())
+		return err
+	}
+	if err := b.CopyBinary(createPipeline); err != nil {
+		gaia.Cfg.Logger.Error("error while copying binary to plugin folder", "error", err.Error())
+		return err
 	}
-	createPipeline.Pipeline = *pipe
-	_ = b.ExecuteBuild(createPipeline)
-	_ = b.CopyBinary(createPipeline)
 	gaia.Cfg.Logger.Debug("successfully updated: ", "message", pipe.Name)
 	return nil
 }
@@ -240,7 +260,8 @@ func NewGithubClient(httpClient *gohttp.Client, repoMock GithubRepoService) Gith
 	}
 }
 
-func createGithubWebhook(token string, repo *gaia.GitRepo, gitRepo GithubRepoService) error {
+func createGithubWebhook(token string, repo *gaia.GitRepo, id string, gitRepo GithubRepoService) error {
+	name := gaia.SecretNamePrefix + id
 	vault, err := services.DefaultVaultService()
 	if err != nil {
 		gaia.Cfg.Logger.Error("unable to initialize vault: ", "error", err.Error())
@@ -260,10 +281,10 @@ func createGithubWebhook(token string, repo *gaia.GitRepo, gitRepo GithubRepoSer
 	tc := oauth2.NewClient(ctx, ts)
 	config := make(map[string]interface{})
 	config["url"] = gaia.Cfg.Hostname + "/api/" + gaia.APIVersion + "/pipeline/githook"
-	secret, err := vault.Get("GITHUB_WEBHOOK_SECRET")
+	secret, err := vault.Get(name)
 	if err != nil {
 		secret = []byte(generateWebhookSecret())
-		vault.Add("GITHUB_WEBHOOK_SECRET", secret)
+		vault.Add(name, secret)
 		err = vault.SaveSecrets()
 		if err != nil {
 			return err