Skip to content

Commit a0b248b

Browse files
manelmontillamanelmontilla
authored
Use read replica (#55)
1 parent 3e7ba64 commit a0b248b

22 files changed

+241
-169
lines changed

‎.travis.yml

+1-4
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,9 @@ env:
88
- CGO_ENABLED=0
99
gobuild_args: -a -tags netgo -ldflags '-w'
1010
go_import_path: github.com/adevinta/vulnerability-db-api
11-
before_script:
12-
- _script/start-pg
1311
script:
1412
- go install ./...
1513
- go test -short -v $(go list ./... | grep -v /vendor/)
16-
- source _script/setup-e2e-tests.sh
17-
- _script/run-e2e-tests.sh
14+
- _script/run-e2e.sh read
1815
after_success:
1916
- bash -c 'source <(curl -s https://raw.githubusercontent.com/adevinta/vulcan-cicd/master/docker.sh)'

‎README.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ For running the component locally, clone and run at the root of the repo the fol
1010
```sh
1111
go install ./...
1212
cd ../vulnerability-db/db && source postgres-start.sh && cd -
13-
cd ../vulnerability-db && source flyway-migrate.sh && cd -
13+
cd ../vulnerability-db/db && source flyway-migrate.sh && cd -
1414
vulnerability-db-api -c ../_resources/config/local.toml
1515
```
1616

@@ -27,8 +27,15 @@ Those are the variables you have to use:
2727
|PG_PASSWORD|Database password|vulnerabilitydb|
2828
|PG_PORT|Database port|5432|
2929
|PG_SSLMODE|One of these (disable,allow,prefer,require,verify-ca,verify-full)|disable|
30+
|PG_HOST_READ|Read replica database host|localhost|
31+
|PG_NAME_READ|Read replica Database name|vulnerabilitydb|
32+
|PG_USER_READ|Read replica Database user|vulnerabilitydb|
33+
|PG_PASSWORD_READ|Read replica Database password|vulnerabilitydb|
34+
|PG_PORT_READ|Read replica Database port|5432|
35+
|PG_SSLMODE_READ|One of these (disable,allow,prefer,require,verify-ca,verify-full)|disable|
3036
|PG_CA_B64|A base64 encoded CA certificate||
3137

38+
3239
```sh
3340
docker build . -t vdba
3441

‎_resources/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44

55
```sh
66
bash postgres-start.sh
7-
psql -h localhost -p 5438 -U vulndb -f schema.sql # Type "vulndb" as password.
7+
psql -h localhost -p 5432 -U vulndb -f schema.sql # Type "vulndb" as password.
88
```
99

1010
## Populating Database
1111

1212
```sh
1313
# Type "vulndb" as password.
14-
psql -h localhost -p 5438 -U vulndb -f schema.sql # Create the schema.
15-
psql -h localhost -p 5438 -U vulndb -f data.sql # Populate with test data.
14+
psql -h localhost -p 5432 -U vulndb -f schema.sql # Create the schema.
15+
psql -h localhost -p 5432 -U vulndb -f data.sql # Populate with test data.
1616
```
1717

1818
## Stopping Database

‎_resources/config/local-e2e.toml ‎_resources/config/e2e-master.toml

+3-2
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ default_size = 20
44

55
[postgresql]
66
host = "localhost"
7-
port = "5440"
7+
port = "5434"
88
user = "vulndb"
99
pass = "vulndb"
1010
db = "vulndbtest"
11+
sslmode = "disable"
1112

1213
[log]
13-
level = "DEBUG"
14+
level = "INFO"

‎_resources/config/e2e-read-replica.toml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[api]
2+
max_size = 100
3+
default_size = 20
4+
5+
[postgresql]
6+
host = "localhost"
7+
port = "5434"
8+
user = "vulndb"
9+
pass = "vulndb"
10+
db = "vulndbtest"
11+
sslmode = "disable"
12+
13+
[postgresql_read]
14+
host = "localhost"
15+
port = "5435"
16+
user = "vulndb"
17+
pass = "vulndb"
18+
db = "vulndbtest"
19+
sslmode = "disable"
20+
21+
[log]
22+
level = "INFO"

‎_resources/docker-compose-e2e.yml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
version: '2'
2+
3+
services:
4+
postgresql-e2e-master:
5+
image: docker.io/bitnami/postgresql:14
6+
ports:
7+
- '5434:5432'
8+
environment:
9+
- POSTGRESQL_REPLICATION_MODE=master
10+
- POSTGRESQL_REPLICATION_USER=repl_user
11+
- POSTGRESQL_REPLICATION_PASSWORD=repl_password
12+
- POSTGRESQL_USERNAME=vulndb
13+
- POSTGRESQL_PASSWORD=vulndb
14+
- POSTGRESQL_DATABASE=vulndbtest
15+
- ALLOW_EMPTY_PASSWORD=yes
16+
healthcheck:
17+
test: ["CMD-SHELL", "sh -c 'pg_isready -U vulndb'"]
18+
interval: 5s
19+
timeout: 10s
20+
retries: 6
21+
22+
postgresql-e2e-slave:
23+
image: docker.io/bitnami/postgresql:14
24+
ports:
25+
- '5435:5432'
26+
depends_on:
27+
postgresql-e2e-master:
28+
condition: service_healthy
29+
environment:
30+
- POSTGRESQL_REPLICATION_MODE=slave
31+
- POSTGRESQL_REPLICATION_USER=repl_user
32+
- POSTGRESQL_REPLICATION_PASSWORD=repl_password
33+
- POSTGRESQL_MASTER_HOST=postgresql-e2e-master
34+
- POSTGRESQL_MASTER_PORT_NUMBER=5432
35+
- ALLOW_EMPTY_PASSWORD=yes
36+
- POSTGRESQL_PASSWORD=vulndb
37+
38+
e2e:
39+
image: postman/newman
40+
command: ["run",
41+
"/postman/Vulnerability_DB_API.postman_collection.json",
42+
"--environment=/postman/VulnerabilityDBAPI.postman_environment.e2e.json",
43+
"--bail",
44+
]
45+
volumes:
46+
- "../_resources/e2e/postman:/postman"
47+
extra_hosts:
48+
- "host.docker.internal:host-gateway"

‎_resources/e2e/postman/VulnerabilityDBAPI.postman_environment.e2e.json

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"id": "670c18fe-cf98-4c48-a725-4866a4ebfdf8",
3+
"name": "VulnerabilityDBAPI",
4+
"values": [
5+
{
6+
"key": "host",
7+
"value": "host.docker.internal:8080",
8+
"type": "default",
9+
"enabled": true
10+
}
11+
],
12+
"_postman_variable_scope": "environment",
13+
"_postman_exported_at": "2022-12-20T11:35:31.612Z",
14+
"_postman_exported_using": "Postman/9.31.0"
15+
}

‎_resources/e2e/postman/Vulnerability_DB_API.postman_collection.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -22722,4 +22722,4 @@
2272222722
"type": "string"
2272322723
}
2272422724
]
22725-
}
22725+
}

‎_script/run-e2e-tests.sh

-9
This file was deleted.

‎_script/run-e2e.sh

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/bash
2+
3+
# Copyright 2022 Adevinta
4+
5+
set -u -e
6+
7+
cd "$(dirname $0)/.."
8+
9+
case $# in
10+
1)
11+
ARG=$1
12+
;;
13+
*)
14+
echo "usage: $0 <read|master>" >&2
15+
exit 2
16+
;;
17+
esac
18+
19+
COMPOSE_OPTIONS=""
20+
case $ARG in
21+
"read")
22+
CONFIG_FILE="_resources/config/e2e-read-replica.toml"
23+
SERVICE="postgresql-e2e-slave"
24+
echo "using the read replica database"
25+
;;
26+
"master")
27+
CONFIG_FILE="_resources/config/e2e-master.toml"
28+
SERVICE="postgresql-e2e-master"
29+
echo "using the master database"
30+
;;
31+
*)
32+
echo "usage: $0 <read|master>" >&2
33+
exit 2
34+
;;
35+
esac
36+
37+
38+
# Start postgress databases.
39+
docker-compose -f _resources/docker-compose-e2e.yml -p "e2e" up -d $SERVICE
40+
41+
# Set up schema for the test db.
42+
set +e
43+
rm -rf /tmp/vulnerability-db
44+
set -e
45+
git clone --single-branch --branch master https://github.com/adevinta/vulnerability-db.git /tmp/vulnerability-db
46+
docker run --net=host -v /tmp/vulnerability-db/db:/flyway/sql flyway/flyway:"${FLYWAY_VERSION:-8}-alpine" \
47+
-user=vulndb -password=vulndb -url=jdbc:postgresql://127.0.0.1:5434/vulndbtest -baselineOnMigrate=true migrate
48+
49+
# Loaf fixtures in the test db.
50+
TestDBPort=5434 go run e2e/fixtures.go _resources/e2e/fixtures/
51+
52+
# Give time to the fixtures data to be replicated to the read replica.
53+
sleep 5
54+
55+
# Kill any possible running server.
56+
set +e
57+
pkill vulnerability-db-api
58+
set -e
59+
60+
echo "RUNNING vulnerability-db-api"
61+
go install ./...
62+
vulnerability-db-api -c $CONFIG_FILE &
63+
64+
# Run e2e tests.
65+
echo "RUNNING e2e tests"
66+
docker-compose -f _resources/docker-compose-e2e.yml -p "e2e" run e2e
67+
68+
# Clean up resurces.
69+
set +e
70+
pkill vulnerability-db-api
71+
set -e
72+
docker-compose -f _resources/docker-compose-e2e.yml -p "e2e" rm -s -f -v
73+

‎_script/setup-e2e-tests.sh

-21
This file was deleted.

‎_script/start-pg

-11
This file was deleted.

‎cmd/vulnerability-db-api/config.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ import (
1414
)
1515

1616
type config struct {
17-
API apiConfig `toml:"api"`
18-
PSQL postgresql.ConnStr `toml:"postgresql"`
19-
Log logConfig `toml:"log"`
17+
API apiConfig `toml:"api"`
18+
PSQL postgresql.ConnStr `toml:"postgresql"`
19+
PSQLRead postgresql.ConnStr `toml:"postgresql_read"`
20+
Log logConfig `toml:"log"`
2021
}
2122

2223
type apiConfig struct {

‎cmd/vulnerability-db-api/main.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,13 @@ func main() {
3030
e := echo.New()
3131

3232
e.Logger.SetLevel(parseLogLvl(config.Log.Level))
33-
34-
db, err := postgresql.NewDB(config.PSQL, e.Logger)
33+
cfgPSQL := config.PSQL
34+
cfgPSQLRead := config.PSQLRead
35+
// If we don't have defined the read replica we use the read write one.
36+
if cfgPSQLRead.Host == "" {
37+
cfgPSQLRead = cfgPSQL
38+
}
39+
db, err := postgresql.NewDB(cfgPSQL, cfgPSQLRead, e.Logger)
3540
if err != nil {
3641
e.Logger.Fatal(err)
3742
}

‎config.toml

+10
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,15 @@ pass = "$PG_PASSWORD"
1010
db = "$PG_NAME"
1111
sslmode = "$PG_SSLMODE"
1212

13+
# Read replica DB. If specified the endpoints that only read data from the DB
14+
# will use this db instance instead of the RW one set above.
15+
[postgresql_read]
16+
host = "$PG_HOST_READ"
17+
port = "$PG_PORT_READ"
18+
user = "$PG_USER_READ"
19+
pass = "$PG_PASSWORD_READ"
20+
db = "$PG_NAME_READ"
21+
sslmode = "$PG_SSLMODE_READ"
22+
1323
[log]
1424
level = "$LOG_LEVEL"

0 commit comments

Comments
 (0)