Provide self-registering storage system

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

cmd/migrate_storage.go

@@ -107,6 +107,8 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
+	goCtx := context.Background()
+
 	if err := storage.Init(); err != nil {
 		return err
 	}
@@ -120,18 +122,23 @@ func runMigrateStorage(ctx *cli.Context) error {
 			log.Fatal("Path must be given when storage is loal")
 			return nil
 		}
-		dstStorage, err = storage.NewLocalStorage(p)
+		dstStorage, err = storage.NewLocalStorage(
+			goCtx,
+			storage.LocalStorageConfig{
+				Path: p,
+			})
 	case setting.MinioStorageType:
 		dstStorage, err = storage.NewMinioStorage(
-			context.Background(),
-			ctx.String("minio-endpoint"),
-			ctx.String("minio-access-key-id"),
-			ctx.String("minio-secret-access-key"),
-			ctx.String("minio-bucket"),
-			ctx.String("minio-location"),
-			ctx.String("minio-base-path"),
-			ctx.Bool("minio-use-ssl"),
-		)
+			goCtx,
+			storage.MinioStorageConfig{
+				Endpoint:        ctx.String("minio-endpoint"),
+				AccessKeyID:     ctx.String("minio-access-key-id"),
+				SecretAccessKey: ctx.String("minio-secret-access-key"),
+				Bucket:          ctx.String("minio-bucket"),
+				Location:        ctx.String("minio-location"),
+				BasePath:        ctx.String("minio-base-path"),
+				UseSSL:          ctx.Bool("minio-use-ssl"),
+			})
 	default:
 		return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage"))
 	}

modules/setting/attachment.go

@@ -44,27 +44,26 @@ func newAttachmentService() {
 			log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type)
 		}
 		Attachment.Storage = storage
+
+		for _, key := range storage.Section.Keys() {
+			if !sec.HasKey(key.Name()) {
+				_, _ = sec.NewKey(key.Name(), key.Value())
+			}
+		}
+		Attachment.Storage.Section = sec
 	}
 
 	// Override
 	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect)
 
-	switch Attachment.Storage.Type {
-	case LocalStorageType:
-		Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
-		if !filepath.IsAbs(Attachment.Path) {
-			Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path)
-		}
-	case MinioStorageType:
-		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint)
-		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID)
-		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey)
-		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket)
-		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location)
-		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL)
-		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
+	Attachment.Storage.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
+	if !filepath.IsAbs(Attachment.Storage.Path) {
+		Attachment.Storage.Path = filepath.Join(AppWorkPath, Attachment.Storage.Path)
+		sec.Key("PATH").SetValue(Attachment.Storage.Path)
 	}
 
+	sec.Key("MINIO_BASE_PATH").MustString("attachments/")
+
 	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
 	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
 	Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)

modules/setting/lfs.go

@@ -48,28 +48,23 @@ func newLFSService() {
 			log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type)
 		}
 		LFS.Storage = storage
+
+		for _, key := range storage.Section.Keys() {
+			if !lfsSec.HasKey(key.Name()) {
+				_, _ = lfsSec.NewKey(key.Name(), key.Value())
+			}
+		}
+		LFS.Storage.Section = lfsSec
 	}
 
 	// Override
 	LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect)
-	switch LFS.Storage.Type {
-	case LocalStorageType:
-		// keep compatible
-		LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
-		LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path)
-		if !filepath.IsAbs(LFS.Path) {
-			LFS.Path = filepath.Join(AppWorkPath, LFS.Path)
-		}
-
-	case MinioStorageType:
-		LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint)
-		LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID)
-		LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey)
-		LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket)
-		LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location)
-		LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL)
-		LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
+	LFS.Storage.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
+	LFS.Storage.Path = lfsSec.Key("PATH").MustString(LFS.Storage.Path)
+	if !filepath.IsAbs(LFS.Storage.Path) {
+		lfsSec.Key("PATH").SetValue(filepath.Join(AppWorkPath, LFS.Storage.Path))
 	}
+	lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
 
 	if LFS.LocksPagingNum == 0 {
 		LFS.LocksPagingNum = 50

modules/setting/storage.go

@@ -5,6 +5,7 @@
 package setting
 
 import (
+	"reflect"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
@@ -21,16 +22,20 @@ const (
 type Storage struct {
 	Type        string
 	Path        string
+	Section     *ini.Section
 	ServeDirect bool
-	Minio       struct {
-		Endpoint        string
-		AccessKeyID     string
-		SecretAccessKey string
-		UseSSL          bool
-		Bucket          string
-		Location        string
-		BasePath        string
+}
+
+// MapTo implements the Mappable interface
+func (s *Storage) MapTo(v interface{}) error {
+	pathValue := reflect.ValueOf(v).FieldByName("Path")
+	if pathValue.IsValid() && pathValue.Kind() == reflect.String {
+		pathValue.SetString(s.Path)
 	}
+	if s.Section != nil {
+		return s.Section.MapTo(v)
+	}
+	return nil
 }
 
 var (
@@ -40,17 +45,14 @@ var (
 func getStorage(sec *ini.Section) Storage {
 	var storage Storage
 	storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType)
+	storage.Section = sec
 	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
-	switch storage.Type {
-	case LocalStorageType:
-	case MinioStorageType:
-		storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
-		storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
-		storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
-		storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
-		storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
-		storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
-	}
+	sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+	sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+	sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+	sec.Key("MINIO_BUCKET").MustString("gitea")
+	sec.Key("MINIO_LOCATION").MustString("us-east-1")
+	sec.Key("MINIO_USE_SSL").MustBool(false)
 	return storage
 }
 

modules/storage/helper.go

@@ -0,0 +1,65 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"encoding/json"
+	"reflect"
+)
+
+// Mappable represents an interface that can MapTo another interface
+type Mappable interface {
+	MapTo(v interface{}) error
+}
+
+// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
+//
+// It will tolerate the cfg being passed as a []byte or string of a json representation of the
+// exemplar or the correct type of the exemplar itself
+func toConfig(exemplar, cfg interface{}) (interface{}, error) {
+
+	// First of all check if we've got the same type as the exemplar - if so it's all fine.
+	if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
+		return cfg, nil
+	}
+
+	// Now if not - does it provide a MapTo function we can try?
+	if mappable, ok := cfg.(Mappable); ok {
+		newVal := reflect.New(reflect.TypeOf(exemplar))
+		if err := mappable.MapTo(newVal.Interface()); err == nil {
+			return newVal.Elem().Interface(), nil
+		}
+		// MapTo has failed us ... let's try the json route ...
+	}
+
+	// OK we've been passed a byte array right?
+	configBytes, ok := cfg.([]byte)
+	if !ok {
+		// oh ... it's a string then?
+		var configStr string
+
+		configStr, ok = cfg.(string)
+		configBytes = []byte(configStr)
+	}
+	if !ok {
+		// hmm ... can we marshal it to json?
+		var err error
+
+		configBytes, err = json.Marshal(cfg)
+		ok = (err == nil)
+	}
+	if !ok {
+		// no ... we've tried hard enough at this point - throw an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg}
+	}
+
+	// OK unmarshal the byte array into a new copy of the exemplar
+	newVal := reflect.New(reflect.TypeOf(exemplar))
+	if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
+		// If we can't unmarshal it then return an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
+	}
+	return newVal.Elem().Interface(), nil
+}

modules/storage/local.go

@@ -5,6 +5,7 @@
 package storage
 
 import (
+	"context"
 	"io"
 	"net/url"
 	"os"
@@ -17,19 +18,35 @@ var (
 	_ ObjectStorage = &LocalStorage{}
 )
 
+// LocalStorageType is the type descriptor for local storage
+const LocalStorageType Type = "local"
+
+// LocalStorageConfig represents the configuration for a local storage
+type LocalStorageConfig struct {
+	Path string `ini:"PATH"`
+}
+
 // LocalStorage represents a local files storage
 type LocalStorage struct {
+	ctx context.Context
 	dir string
 }
 
 // NewLocalStorage returns a local files
-func NewLocalStorage(bucket string) (*LocalStorage, error) {
-	if err := os.MkdirAll(bucket, os.ModePerm); err != nil {
+func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
+	configInterface, err := toConfig(LocalStorageConfig{}, cfg)
+	if err != nil {
+		return nil, err
+	}
+	config := configInterface.(LocalStorageConfig)
+
+	if err := os.MkdirAll(config.Path, os.ModePerm); err != nil {
 		return nil, err
 	}
 
 	return &LocalStorage{
-		dir: bucket,
+		ctx: ctx,
+		dir: config.Path,
 	}, nil
 }
 
@@ -80,6 +97,11 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
 		if err != nil {
 			return err
 		}
+		select {
+		case <-l.ctx.Done():
+			return l.ctx.Err()
+		default:
+		}
 		if path == l.dir {
 			return nil
 		}
@@ -98,3 +120,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
 		return fn(relPath, obj)
 	})
 }
+
+func init() {
+	RegisterStorageType(LocalStorageType, NewLocalStorage)
+}