Skip to content

Commit 8423c70

Browse files
sgottisgotti
committed
*: new db schema
* Remove query tables. Use a standard sql approach storing all possible fields as columns. * Some fields that are complex (like runtasks) are saved as json. * For postgres use native sequences. For sqlite3 emulate them with a sequence table. * Add sqlg package that provides code generators, db, lock and a db manager for setup/migration. * Future migrations could be done with the new migration logic and won't require a conversion from an export but will be done directly on the database. * Added migration command to migrate from an export of the db schema available in master (at commit 248a9e0) in addition to the one already available to migrate from v0.7.x. To define if the source export is the one at v0.7.x or at commit 248a9e0 the migrate command provides a "--source-version" option. * Run test for both sqlite3 and postgres databases * Currently integration tests are run only for sqlite3
1 parent 8859b18 commit 8423c70

File tree

144 files changed

+20956
-6447
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

144 files changed

+20956
-6447
lines changed

‎.agola/config.jsonnet

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ local go_runtime(version, arch) = {
55
{
66
image: 'golang:' + version + '-buster',
77
},
8+
{
9+
image: 'postgres',
10+
environment: {
11+
POSTGRES_PASSWORD: 'password',
12+
},
13+
},
814
],
915
};
1016

@@ -36,7 +42,21 @@ local task_build_go(version, arch) = {
3642
{ type: 'run', command: 'golangci-lint run --deadline 5m' },
3743
{ type: 'run', name: 'build docker/k8s drivers tests binary', command: 'CGO_ENABLED=0 go test -c ./internal/services/executor/driver -o ./bin/docker-tests' },
3844
{ type: 'run', name: 'build integration tests binary', command: 'go test -tags "sqlite_unlock_notify" -c ./tests -o ./bin/integration-tests' },
39-
{ type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
45+
{ type: 'run', name: 'run tests (sqlite3)',
46+
environment: {
47+
DB_TYPE: "sqlite3",
48+
SKIP_DOCKER_TESTS: "1",
49+
SKIP_K8S_TESTS: "1",
50+
},
51+
command: 'go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
52+
{ type: 'run', name: 'run tests (postgres)',
53+
environment: {
54+
DB_TYPE: "postgres",
55+
PG_CONNSTRING: "postgres://postgres:postgres@localhost/%s?sslmode=disable",
56+
SKIP_DOCKER_TESTS: "1",
57+
SKIP_K8S_TESTS: "1",
58+
},
59+
command: 'go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
4060
{ type: 'run', name: 'fetch gitea binary for integration tests', command: 'curl -L https://github.com/go-gitea/gitea/releases/download/v1.15.11/gitea-1.15.11-linux-amd64 -o ./bin/gitea && chmod +x ./bin/gitea' },
4161
{ type: 'save_to_workspace', contents: [{ source_dir: './bin', dest_dir: '/bin/', paths: ['*'] }] },
4262
],

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,11 @@ webbundle/bindata.go: go-bindata $(WEBDISTPATH)
6060

6161
.PHONY: generate
6262
generate: generators
63-
go generate ./...
63+
go generate $(PROJDIR)/...
6464

6565
.PHONY: generators
6666
generators:
67-
GOBIN=$(PROJDIR)/tools/bin go install ./internal/generators
67+
go build -o $(PROJDIR)/tools/bin/dbgenerator ./internal/generators/db
6868

6969
.PHONY: docker-agola
7070
docker-agola:

‎cmd/agola/cmd/migrate.go

Lines changed: 80 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,22 @@ package cmd
1616

1717
import (
1818
"context"
19-
"os"
2019

2120
"github.com/rs/zerolog/log"
2221
"github.com/sorintlab/errors"
2322
"github.com/spf13/cobra"
2423

25-
"agola.io/agola/internal/migration"
24+
"agola.io/agola/internal/services/config"
25+
csdb "agola.io/agola/internal/services/configstore/db"
26+
rsdb "agola.io/agola/internal/services/runservice/db"
27+
"agola.io/agola/internal/sqlg/lock"
28+
"agola.io/agola/internal/sqlg/manager"
29+
"agola.io/agola/internal/sqlg/sql"
2630
)
2731

2832
var cmdMigrate = &cobra.Command{
2933
Use: "migrate",
30-
Short: "migrate from an old data format export to the new data format",
34+
Short: "migrate component database to latest version",
3135
Run: func(cmd *cobra.Command, args []string) {
3236
if err := migrate(cmd, args); err != nil {
3337
log.Fatal().Err(err).Send()
@@ -36,61 +40,107 @@ var cmdMigrate = &cobra.Command{
3640
}
3741

3842
type migrateOptions struct {
43+
config string
3944
serviceName string
40-
inFilePath string
41-
outFilePath string
4245
}
4346

4447
var migrateOpts migrateOptions
4548

4649
func init() {
4750
flags := cmdMigrate.Flags()
4851

52+
flags.StringVar(&migrateOpts.config, "config", "./config.yml", "config file path")
4953
flags.StringVar(&migrateOpts.serviceName, "service", "", "service name (runservice or configstore)")
50-
flags.StringVar(&migrateOpts.inFilePath, "in", "-", "input file path")
51-
flags.StringVar(&migrateOpts.outFilePath, "out", "-", "output file path")
5254

5355
cmdAgola.AddCommand(cmdMigrate)
5456
}
5557

5658
func migrate(cmd *cobra.Command, args []string) error {
59+
ctx := context.Background()
60+
5761
if migrateOpts.serviceName != "runservice" && migrateOpts.serviceName != "configstore" {
5862
return errors.Errorf("service option must be runservice or configstore")
5963
}
6064

61-
var r *os.File
62-
if migrateOpts.inFilePath == "-" {
63-
r = os.Stdin
64-
} else {
65-
var err error
66-
r, err = os.Open(migrateOpts.inFilePath)
67-
if err != nil {
68-
return errors.WithStack(err)
69-
}
65+
components := []string{migrateOpts.serviceName}
66+
67+
c, err := config.Parse(migrateOpts.config, components)
68+
if err != nil {
69+
return errors.Wrapf(err, "config error")
7070
}
7171

72-
var w *os.File
73-
if migrateOpts.outFilePath == "-" {
74-
w = os.Stdout
75-
} else {
72+
var sdb *sql.DB
73+
var d manager.DB
74+
switch migrateOpts.serviceName {
75+
case "runservice":
7676
var err error
77-
w, err = os.Create(migrateOpts.outFilePath)
77+
78+
dbConf := c.Runservice.DB
79+
80+
sdb, err = sql.NewDB(dbConf.Type, dbConf.ConnString)
7881
if err != nil {
79-
return errors.WithStack(err)
82+
return errors.Wrapf(err, "new db error")
8083
}
81-
}
8284

83-
log.Info().Msgf("migrating %s", migrateOpts.serviceName)
84-
switch migrateOpts.serviceName {
85-
case "runservice":
86-
if err := migration.MigrateRunService(context.Background(), r, w); err != nil {
87-
return errors.WithStack(err)
85+
d, err = rsdb.NewDB(log.Logger, sdb)
86+
if err != nil {
87+
return errors.Wrapf(err, "new db error")
8888
}
89+
8990
case "configstore":
90-
if err := migration.MigrateConfigStore(context.Background(), r, w); err != nil {
91-
return errors.WithStack(err)
91+
var err error
92+
93+
dbConf := c.Configstore.DB
94+
95+
sdb, err = sql.NewDB(dbConf.Type, dbConf.ConnString)
96+
if err != nil {
97+
return errors.Wrapf(err, "new db error")
9298
}
99+
100+
d, err = csdb.NewDB(log.Logger, sdb)
101+
if err != nil {
102+
return errors.Wrapf(err, "new db error")
103+
}
104+
}
105+
106+
var lf lock.LockFactory
107+
switch d.DBType() {
108+
case sql.Sqlite3:
109+
ll := lock.NewLocalLocks()
110+
lf = lock.NewLocalLockFactory(ll)
111+
case sql.Postgres:
112+
lf = lock.NewPGLockFactory(sdb)
113+
default:
114+
return errors.Errorf("unknown db type %q", d.DBType())
115+
}
116+
117+
dbm := manager.NewDBManager(log.Logger, d, lf)
118+
119+
log.Info().Msgf("migrating service %s", migrateOpts.serviceName)
120+
121+
curDBVersion, err := dbm.GetVersion(ctx)
122+
if err != nil {
123+
return errors.WithStack(err)
93124
}
94125

126+
if err := dbm.CheckVersion(curDBVersion, d.Version()); err != nil {
127+
return errors.WithStack(err)
128+
}
129+
130+
migrationRequired, err := dbm.CheckMigrationRequired(curDBVersion, d.Version())
131+
if err != nil {
132+
return errors.WithStack(err)
133+
}
134+
if !migrationRequired {
135+
log.Info().Msgf("db already at latest version: %d", curDBVersion)
136+
return nil
137+
}
138+
139+
if err := dbm.Migrate(ctx); err != nil {
140+
return errors.Wrap(err, "migrate db error")
141+
}
142+
143+
log.Info().Msgf("db migrated to version: %d", d.Version())
144+
95145
return nil
96146
}

‎cmd/agola/cmd/migrateexport.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright 2022 Sorint.lab
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package cmd
16+
17+
import (
18+
"context"
19+
"os"
20+
21+
"github.com/rs/zerolog/log"
22+
"github.com/sorintlab/errors"
23+
"github.com/spf13/cobra"
24+
25+
migration248a9e0ad "agola.io/agola/internal/migration/248a9e0ad"
26+
migrationv07x "agola.io/agola/internal/migration/v0.7.x"
27+
)
28+
29+
var cmdMigrateExport = &cobra.Command{
30+
Use: "migrateexport",
31+
Short: "migrate from an old data format export to the new data format",
32+
Run: func(cmd *cobra.Command, args []string) {
33+
if err := migrateExport(cmd, args); err != nil {
34+
log.Fatal().Err(err).Send()
35+
}
36+
},
37+
}
38+
39+
type migrateExportOptions struct {
40+
serviceName string
41+
sourceVersion string
42+
inFilePath string
43+
outFilePath string
44+
}
45+
46+
var migrateExportOpts migrateExportOptions
47+
48+
func init() {
49+
flags := cmdMigrateExport.Flags()
50+
51+
flags.StringVar(&migrateExportOpts.serviceName, "service", "", "service name (runservice or configstore)")
52+
flags.StringVar(&migrateExportOpts.sourceVersion, "source-version", "v0.7.x", "export source version (v0.7.x or 248a9e0ad)")
53+
flags.StringVar(&migrateExportOpts.inFilePath, "in", "-", "input file path")
54+
flags.StringVar(&migrateExportOpts.outFilePath, "out", "-", "output file path")
55+
56+
cmdAgola.AddCommand(cmdMigrateExport)
57+
}
58+
59+
func migrateExport(cmd *cobra.Command, args []string) error {
60+
if migrateExportOpts.serviceName != "runservice" && migrateExportOpts.serviceName != "configstore" {
61+
return errors.Errorf("service option must be runservice or configstore")
62+
}
63+
if migrateExportOpts.sourceVersion != "v0.7.x" && migrateExportOpts.sourceVersion != "248a9e0ad" {
64+
return errors.Errorf("source version option must be v0.7.x or 248a9e0ad")
65+
}
66+
67+
var r *os.File
68+
if migrateExportOpts.inFilePath == "-" {
69+
r = os.Stdin
70+
} else {
71+
var err error
72+
r, err = os.Open(migrateExportOpts.inFilePath)
73+
if err != nil {
74+
return errors.WithStack(err)
75+
}
76+
}
77+
78+
var w *os.File
79+
if migrateExportOpts.outFilePath == "-" {
80+
w = os.Stdout
81+
} else {
82+
var err error
83+
w, err = os.Create(migrateExportOpts.outFilePath)
84+
if err != nil {
85+
return errors.WithStack(err)
86+
}
87+
}
88+
89+
log.Info().Msgf("migrating export of service %s", migrateExportOpts.serviceName)
90+
switch migrateExportOpts.serviceName {
91+
case "runservice":
92+
switch migrateExportOpts.sourceVersion {
93+
case "v0.7.x":
94+
if err := migrationv07x.MigrateRunService(context.Background(), r, w); err != nil {
95+
return errors.WithStack(err)
96+
}
97+
case "248a9e0ad":
98+
if err := migration248a9e0ad.MigrateRunService(context.Background(), r, w); err != nil {
99+
return errors.WithStack(err)
100+
}
101+
}
102+
case "configstore":
103+
switch migrateExportOpts.sourceVersion {
104+
case "v0.7.x":
105+
if err := migrationv07x.MigrateConfigStore(context.Background(), r, w); err != nil {
106+
return errors.WithStack(err)
107+
}
108+
case "248a9e0ad":
109+
if err := migration248a9e0ad.MigrateConfigStore(context.Background(), r, w); err != nil {
110+
return errors.WithStack(err)
111+
}
112+
}
113+
}
114+
115+
return nil
116+
}

‎doc/migrating_from_v0.7.x.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ We suggest to test this migration on a test environment before doing this on you
1515

1616
`curl -v http://$CONFIGSTOREHOST:PORT/api/v1alpha/export > /tmp/configstore-export`
1717

18-
1. Generate the migrated data using the new agola binary migrate command:
18+
1. Generate the migrated data using the new agola binary `migrateexport` command:
1919

20-
`cat /tmp/runservice-export | ./tmp/agola migrate --service runservice > /tmp/runservice-migrated`
20+
`cat /tmp/runservice-export | ./tmp/agola migrateexport --service runservice > /tmp/runservice-migrated`
2121

22-
`cat /tmp/runservice-export | ./tmp/agola migrate --service configstore > /tmp/configstore-migrated`
22+
`cat /tmp/runservice-export | ./tmp/agola migratexporte --service configstore > /tmp/configstore-migrated`
2323

2424
1. Update the agola binaries on your environment or use a test enviroment and start only the runservice and configstore.
2525
1. Update the agola config file and remove the runservice, configstore, notification service etcd entries and add the db entries. Every component should have its own dedicated database. DO NOT use the same database for all the services. For PostgresSQL it can be the same postgres instance but with different databases.
@@ -42,3 +42,7 @@ We suggest to test this migration on a test environment before doing this on you
4242
`curl -v -XDELETE http://$NEWCONFIGSTOREHOST:PORT/api/v1alpha/maintenance`
4343

4444
1. Start the gateway and test if the migration was successfull
45+
46+
## Migrating from db after commit 248a9e0ad and before v0.8.x
47+
48+
Use the same steps but provide the `migrateexport` option `--source-version 248a9e0ad`

0 commit comments

Comments
 (0)