Working polling system with building all pipelines.

Author: Skarlso

Makefile

@@ -4,7 +4,7 @@ GO_LDFLAGS_STATIC=-ldflags "-s -w -extldflags -static"
 default: dev
 
 dev:
-	go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev true
+	go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev=true -poll=true
 
 compile_frontend:
 	cd ./frontend && \

cmd/gaia/main.go

@@ -37,7 +37,7 @@ func init() {
 	flag.StringVar(&gaia.Cfg.Worker, "worker", "2", "Number of worker gaia will use to execute pipelines in parallel")
 	flag.BoolVar(&gaia.Cfg.DevMode, "dev", false, "If true, gaia will be started in development mode. Don't use this in production!")
 	flag.BoolVar(&gaia.Cfg.VersionSwitch, "version", false, "If true, will print the version and immediately exit")
-
+	flag.BoolVar(&gaia.Cfg.Poll, "poll", false, "Instead of using a Webhook, keep polling git for changes on pipelines")
 	// Default values
 	gaia.Cfg.Bolt.Mode = 0600
 }

gaia.go

@@ -148,6 +148,7 @@ var Cfg *Config
 type Config struct {
 	DevMode       bool
 	VersionSwitch bool
+	Poll          bool
 	ListenPort    string
 	HomePath      string
 	DataPath      string

pipeline/build_golang.go

@@ -41,7 +41,7 @@ func (b *BuildPipelineGolang) PrepareEnvironment(p *gaia.CreatePipeline) error {
 	// Set new generated path in pipeline obj for later usage
 	p.Pipeline.Repo.LocalDest = cloneFolder
 	p.Pipeline.UUID = uuid.String()
-	return nil
+	return err
 }
 
 // ExecuteBuild executes the golang build process
@@ -116,6 +116,12 @@ func (b *BuildPipelineGolang) CopyBinary(p *gaia.CreatePipeline) error {
 		return err
 	}
 
+	p.Pipeline.ExecPath = dest
+	// Our pipeline is finished constructing. Save it.
+	if err := storeService.PipelinePut(&p.Pipeline); err != nil {
+		return err
+	}
+
 	// Set +x (execution right) for pipeline
 	return os.Chmod(dest, 0766)
 }

pipeline/build_golang_test.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/gaia-pipeline/gaia/store"
+
 	"github.com/gaia-pipeline/gaia"
 	hclog "github.com/hashicorp/go-hclog"
 )
@@ -181,6 +183,10 @@ func TestExecuteBuildBinaryNotFoundError(t *testing.T) {
 }
 
 func TestCopyBinary(t *testing.T) {
+	s := store.NewStore()
+	s.Init()
+	storeService = s
+	defer os.Remove("gaia.db")
 	tmp := os.TempDir()
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp

pipeline/git.go

@@ -1,7 +1,6 @@
 package pipeline
 
 import (
-	"path/filepath"
 	"strings"
 	"sync"
 
@@ -117,16 +116,15 @@ func updateAllCurrentPipelines() {
 	for pipeline := range GlobalActivePipelines.Iter() {
 		allPipelines = append(allPipelines, pipeline)
 	}
-	goPath := filepath.Join(gaia.Cfg.HomePath, tmpFolder, golangFolder)
 	for _, p := range allPipelines {
 		wg.Add(1)
 		go func(pipe gaia.Pipeline) {
 			defer wg.Done()
 			sem <- 1
-			cloneFolder := filepath.Join(goPath, srcFolder, pipe.UUID)
-			r, err := git.PlainOpen(cloneFolder)
+			r, err := git.PlainOpen(pipe.Repo.LocalDest)
 			if err != nil {
-				// ignore for now
+				// It's also an error if the repo is already up to date so we just move on.
+				gaia.Cfg.Logger.Debug("error while opening repo: ", pipe.Repo.LocalDest, err.Error())
 				return
 			}
 			beforPull, _ := r.Head()

pipeline/ticker.go

@@ -54,16 +54,22 @@ func InitTicker(store *store.Store, scheduler *scheduler.Scheduler) {
 		}
 	}()
 
-	pollTicket := time.NewTicker(pollTicketIntervalMinutes * time.Minute)
-	go func() {
-		defer pollTicket.Stop()
-		for {
-			select {
-			case <-pollTicket.C:
-				updateAllCurrentPipelines()
+	if gaia.Cfg.Poll {
+		pollTicket := time.NewTicker(pollTicketIntervalMinutes * time.Minute)
+		go func() {
+			defer pollTicket.Stop()
+			for {
+				select {
+				case <-pollTicket.C:
+					updateAllCurrentPipelines()
+				}
 			}
-		}
-	}()
+		}()
+	} else {
+		gaia.Cfg.Logger.Debug("webhooks enabled")
+		// WebHook needs to be an option on the UI for
+		// a registered repository. If polling is on, that checkbox needs to be disabled.
+	}
 }
 
 // checkActivePipelines looks up all files in the pipeline folder.