Added ticker as a periodic job to check pipelines folder for new pipelines

Author: michelvocks

cmd/gaia/main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/gaia-pipeline/gaia"
 	"github.com/gaia-pipeline/gaia/handlers"
+	"github.com/gaia-pipeline/gaia/pipeline"
 	"github.com/gaia-pipeline/gaia/store"
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/kataras/iris"
@@ -16,6 +17,11 @@ var (
 	irisInstance *iris.Application
 )
 
+const (
+	dataFolder      = "data"
+	pipelinesFolder = "pipelines"
+)
+
 func init() {
 	gaia.Cfg = &gaia.Config{}
 
@@ -51,12 +57,22 @@ func main() {
 		gaia.Cfg.Logger.Debug("executeable path found", "path", execPath)
 	}
 
+	// Set data path and pipeline path relative to home folder and create it
+	// if not exist.
+	gaia.Cfg.DataPath = gaia.Cfg.HomePath + string(os.PathSeparator) + dataFolder
+	gaia.Cfg.PipelinePath = gaia.Cfg.DataPath + string(os.PathSeparator) + pipelinesFolder
+	err := os.MkdirAll(gaia.Cfg.PipelinePath, 0700)
+	if err != nil {
+		gaia.Cfg.Logger.Error("cannot create data folder", "error", err.Error(), "path", gaia.Cfg.DataPath)
+		os.Exit(1)
+	}
+
 	// Initialize IRIS
 	irisInstance = iris.New()
 
 	// Initialize store
 	s := store.NewStore()
-	err := s.Init()
+	err = s.Init()
 	if err != nil {
 		gaia.Cfg.Logger.Error("cannot initialize store", "error", err.Error())
 		os.Exit(1)
@@ -69,6 +85,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Start ticker. Periodic job to check for new plugins.
+	pipeline.InitTicker()
+
 	// Start listen
 	irisInstance.Run(iris.Addr(":" + gaia.Cfg.ListenPort))
 }

frontend/client/views/pipelines/create.vue

@@ -399,7 +399,7 @@ export default {
 }
 
 .progress-bar-height {
-  height: 45px;
+  height: 50px;
 }
 
 .table td {

gaia.go

@@ -65,9 +65,11 @@ var Cfg *Config
 
 // Config holds all config options
 type Config struct {
-	ListenPort string
-	HomePath   string
-	Logger     hclog.Logger
+	ListenPort   string
+	HomePath     string
+	DataPath     string
+	PipelinePath string
+	Logger       hclog.Logger
 
 	Bolt struct {
 		Path string

handlers/pipeline.go

@@ -60,7 +60,6 @@ func CreatePipeline(ctx iris.Context) {
 		ctx.WriteString(err.Error())
 		return
 	}
-	gaia.Cfg.Logger.Debug("create pipeline", "pipeline", (*p))
 
 	// Set the creation date and unique id
 	p.Created = time.Now()

pipeline/build_golang.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"context"
+	"io"
 	"os"
 	"os/exec"
 	"time"
@@ -102,6 +103,33 @@ func executeCmd(path string, args []string, env []string, dir string) ([]byte, e
 // CopyBinary copies the final compiled archive to the
 // destination folder.
 func (b *BuildPipelineGolang) CopyBinary(p *gaia.CreatePipeline) error {
-	// TODO
-	return nil
+	// Define src and destination
+	src := p.Pipeline.Repo.LocalDest + string(os.PathSeparator) + p.Pipeline.Name
+	dest := gaia.Cfg.PipelinePath + string(os.PathSeparator) + p.Pipeline.Name
+
+	return copyFileContents(src, dest)
+}
+
+// copyFileContents copies the content from source to destination.
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		cerr := out.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+	err = out.Sync()
+	return
 }

pipeline/ticker.go

@@ -0,0 +1,43 @@
+package pipeline
+
+import (
+	"io/ioutil"
+	"time"
+
+	"github.com/gaia-pipeline/gaia"
+)
+
+const (
+	tickerIntervalSeconds = 5
+)
+
+// InitTicker inititates the pipeline ticker.
+// This periodic job will check for new pipelines.
+func InitTicker() {
+	// Create ticker
+	ticker := time.NewTicker(tickerIntervalSeconds * time.Second)
+	quit := make(chan struct{})
+
+	// Actual ticker implementation
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				files, err := ioutil.ReadDir(gaia.Cfg.PipelinePath)
+				if err != nil {
+					gaia.Cfg.Logger.Error("cannot read pipelines folder", "error", err.Error(), "path", gaia.Cfg.PipelinePath)
+				} else {
+					// Iterate all found pipeline
+					for _, file := range files {
+						// TODO: Create for every file a pipeline object
+						// and store it in a global pipeline array
+						gaia.Cfg.Logger.Debug("pipeline found", "name", file.Name())
+					}
+				}
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}

store/store.go

@@ -8,10 +8,6 @@ import (
 	"github.com/gaia-pipeline/gaia"
 )
 
-const (
-	dataFolder = "data"
-)
-
 var (
 	// Name of the bucket where we store user objects
 	userBucket = []byte("Users")
@@ -45,15 +41,8 @@ func NewStore() *Store {
 // This should be called only once per database
 // because bolt holds a lock on the database file.
 func (s *Store) Init() error {
-	// Make sure data folder exists
-	folder := gaia.Cfg.HomePath + string(os.PathSeparator) + dataFolder
-	err := os.MkdirAll(folder, 0700)
-	if err != nil {
-		return err
-	}
-
 	// Open connection to bolt database
-	path := folder + string(os.PathSeparator) + gaia.Cfg.Bolt.Path
+	path := gaia.Cfg.DataPath + string(os.PathSeparator) + gaia.Cfg.Bolt.Path
 	db, err := bolt.Open(path, gaia.Cfg.Bolt.Mode, nil)
 	if err != nil {
 		return err