Python create pipeline race condition (#132)

* Changed order of pipeline creation process

* Added execpath set after build execution

* Removed manual set for execpath and tested race condition fix

* Fixed failing test

Author: michelvocks

frontend/client/views/pipeline/create.vue

@@ -371,12 +371,12 @@ export default {
 
                 if (this.historyRows[i].status >= 50) {
                   this.historyRows[i].output += 'Pipeline has been successfully compiled.\n'
-                  this.historyRows[i].output += 'Copy binary to pipelines folder...\n'
+                  this.historyRows[i].output += 'Verifying pipeline...\n'
                 }
 
                 if (this.historyRows[i].status >= 75) {
-                  this.historyRows[i].output += 'Pipeline binary has been copied to pipelines folder.\n'
-                  this.historyRows[i].output += 'Starting integrity check...\n'
+                  this.historyRows[i].output += 'Pipeline has been verified.\n'
+                  this.historyRows[i].output += 'Finalizing creation process...\n'
                 }
 
                 if (this.historyRows[i].status === 100) {

workers/pipeline/build_cpp.go

@@ -60,6 +60,10 @@ func (b *BuildPipelineCpp) ExecuteBuild(p *gaia.CreatePipeline) error {
 		return err
 	}
 
+	// Build has been finished. Set execution path to the build result archive.
+	// This will be used during pipeline verification phase which will happen after this step.
+	p.Pipeline.ExecPath = filepath.Join(p.Pipeline.Repo.LocalDest, cppFinalBinaryName)
+
 	return nil
 }
 

workers/pipeline/build_golang.go

@@ -82,6 +82,10 @@ func (b *BuildPipelineGolang) ExecuteBuild(p *gaia.CreatePipeline) error {
 		return err
 	}
 
+	// Build has been finished. Set execution path to the build result archive.
+	// This will be used during pipeline verification phase which will happen after this step.
+	p.Pipeline.ExecPath = filepath.Join(p.Pipeline.Repo.LocalDest, appendTypeToName(p.Pipeline.Name, p.Pipeline.Type))
+
 	return nil
 }
 

workers/pipeline/build_java.go

@@ -71,6 +71,10 @@ func (b *BuildPipelineJava) ExecuteBuild(p *gaia.CreatePipeline) error {
 		return err
 	}
 
+	// Build has been finished. Set execution path to the build result archive.
+	// This will be used during pipeline verification phase which will happen after this step.
+	p.Pipeline.ExecPath = filepath.Join(p.Pipeline.Repo.LocalDest, mavenTargetFolder, javaFinalJarName)
+
 	return nil
 }
 

workers/pipeline/build_python.go

@@ -65,17 +65,25 @@ func (b *BuildPipelinePython) ExecuteBuild(p *gaia.CreatePipeline) error {
 		return err
 	}
 
+	// Build has been finished. Set execution path to the build result archive.
+	// This will be used during pipeline verification phase which will happen after this step.
+	p.Pipeline.ExecPath, err = findPythonArchivePath(p)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
-// CopyBinary copies the final compiled archive to the
-// destination folder.
-func (b *BuildPipelinePython) CopyBinary(p *gaia.CreatePipeline) error {
+// findPythonArchivePath filters the archives in the generated dist folder
+// and looks for the final archive. It will return an error if less or more
+// than one file(s) are found otherwise the full path to the file.
+func findPythonArchivePath(p *gaia.CreatePipeline) (src string, err error) {
 	// find all files in dist folder
 	distFolder := filepath.Join(p.Pipeline.Repo.LocalDest, "dist")
 	files, err := ioutil.ReadDir(distFolder)
 	if err != nil {
-		return err
+		return
 	}
 
 	// filter for archives
@@ -88,12 +96,24 @@ func (b *BuildPipelinePython) CopyBinary(p *gaia.CreatePipeline) error {
 
 	// if we found more or less than one archive we have a problem
 	if len(archive) != 1 {
-		gaia.Cfg.Logger.Debug("cannot copy python package", "foundPackages", len(archive), "archives", files)
-		return errors.New("cannot copy python package: not found")
+		gaia.Cfg.Logger.Debug("cannot find python package", "foundPackages", len(archive), "archives", files)
+		err = errors.New("cannot find python package")
+		return
 	}
 
+	// Return full path
+	src = filepath.Join(distFolder, archive[0].Name())
+	return
+}
+
+// CopyBinary copies the final compiled archive to the
+// destination folder.
+func (b *BuildPipelinePython) CopyBinary(p *gaia.CreatePipeline) error {
 	// Define src and destination
-	src := filepath.Join(distFolder, archive[0].Name())
+	src, err := findPythonArchivePath(p)
+	if err != nil {
+		return err
+	}
 	dest := filepath.Join(gaia.Cfg.PipelinePath, appendTypeToName(p.Pipeline.Name, p.Pipeline.Type))
 
 	// Copy binary

workers/pipeline/build_python_test.go

@@ -51,13 +51,22 @@ func TestExecuteBuildPython(t *testing.T) {
 	tmp, _ := ioutil.TempDir("", "TestExecuteBuildPython")
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
-	b := new(BuildPipelinePython)
 	p := new(gaia.CreatePipeline)
+	p.Pipeline.Name = "main"
+	p.Pipeline.Type = gaia.PTypePython
+	p.Pipeline.Repo.LocalDest = tmp
+	os.Mkdir(filepath.Join(tmp, "dist"), 0744)
+	src := filepath.Join(tmp, "dist", p.Pipeline.Name+".tar.gz")
+	f, _ := os.Create(src)
+	defer os.RemoveAll(tmp)
+	defer f.Close()
+	ioutil.WriteFile(src, []byte("testcontent"), 0666)
+	b := new(BuildPipelinePython)
 	// go must be existent, python maybe not.
 	pythonBinaryName = "go"
 	err := b.ExecuteBuild(p)
 	if err != nil {
-		t.Fatal("error while running executebuild. none was expected")
+		t.Fatalf("error while running executebuild. none was expected but got %s", err.Error())
 	}
 	expectedBuildArgs := "setup.py,sdist"
 	actualArgs := os.Getenv("CMD_ARGS")

workers/pipeline/create_pipeline.go

@@ -2,7 +2,6 @@ package pipeline
 
 import (
 	"fmt"
-	"path/filepath"
 
 	"github.com/gaia-pipeline/gaia"
 	"github.com/gaia-pipeline/gaia/services"
@@ -15,8 +14,8 @@ const (
 	// Percent of pipeline creation progress after compile process done
 	pipelineCompileStatus = 50
 
-	// Percent of pipeline creation progress after validation binary copy
-	pipelineCopyStatus = 75
+	// Percent of pipeline creation progress after validation
+	pipelineValidateStatus = 75
 
 	// Completed percent progress
 	pipelineCompleteStatus = 100
@@ -81,33 +80,12 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 		return
 	}
 
-	// Copy compiled binary to plugins folder
-	err = bP.CopyBinary(p)
-	if err != nil {
-		p.StatusType = gaia.CreatePipelineFailed
-		p.Output = fmt.Sprintf("cannot copy compiled binary: %s", err.Error())
-		storeService.CreatePipelinePut(p)
-		return
-	}
-
-	// Update status of our pipeline build
-	p.Status = pipelineCopyStatus
-	err = storeService.CreatePipelinePut(p)
-	if err != nil {
-		gaia.Cfg.Logger.Error("cannot put create pipeline into store", "error", err.Error())
-		return
-	}
-
 	// Run update if needed
-	p.Pipeline.ExecPath = filepath.Join(gaia.Cfg.PipelinePath, appendTypeToName(p.Pipeline.Name, p.Pipeline.Type))
 	err = updatePipeline(&p.Pipeline)
 	if err != nil {
 		p.StatusType = gaia.CreatePipelineFailed
 		p.Output = fmt.Sprintf("cannot update pipeline: %s", err.Error())
 		storeService.CreatePipelinePut(p)
-
-		// Creation failed. Remove broken pipeline.
-		DeleteBinary(p.Pipeline)
 		return
 	}
 
@@ -117,9 +95,14 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 		p.StatusType = gaia.CreatePipelineFailed
 		p.Output = fmt.Sprintf("cannot validate pipeline: %s", err.Error())
 		storeService.CreatePipelinePut(p)
+		return
+	}
 
-		// Creation failed. Remove broken pipeline.
-		DeleteBinary(p.Pipeline)
+	// Update status of our pipeline build
+	p.Status = pipelineValidateStatus
+	err = storeService.CreatePipelinePut(p)
+	if err != nil {
+		gaia.Cfg.Logger.Error("cannot put create pipeline into store", "error", err.Error())
 		return
 	}
 
@@ -132,6 +115,15 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 		return
 	}
 
+	// Copy compiled binary to plugins folder which is the final step
+	err = bP.CopyBinary(p)
+	if err != nil {
+		p.StatusType = gaia.CreatePipelineFailed
+		p.Output = fmt.Sprintf("cannot copy compiled binary: %s", err.Error())
+		storeService.CreatePipelinePut(p)
+		return
+	}
+
 	// Set create pipeline status to complete
 	p.Status = pipelineCompleteStatus
 	p.StatusType = gaia.CreatePipelineSuccess