Skip to content

Commit 29864ca

Browse files
atercattusatercattus
authored
Merge pull request #559 from go-mysql-org/atercattus/migrate-to-github-actions
* added ci.yml; removed .travis.yml * dump - add mysqldump --column-statistics parameter detection * update go.mod * update go.sum * added .golangci.yml * lint - fix all misspell * lint - fix all staticcheck * lint - fix all errcheck * lint - fix all structcheck * lint - fix all nakedret * lint - fix all goimports * lint - fix all unconvert * lint - fix all whitespace
2 parents 0c5789d + 52a70ec commit 29864ca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1335
-1054
lines changed

‎.github/workflows/ci.yml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: runTestsAndLinters
2+
on: [push, pull_request]
3+
4+
jobs:
5+
test:
6+
strategy:
7+
matrix:
8+
go: [ 1.16, 1.15 ]
9+
name: Tests Go ${{ matrix.go }}
10+
runs-on: ubuntu-18.04
11+
12+
steps:
13+
- name: Setup MySQL
14+
run: |
15+
echo -n "mysql -V: " ; mysql -V
16+
echo -n "mysqldump -V: " ; mysqldump -V
17+
18+
echo -e '[mysqld]\nserver-id=1\nlog-bin=mysql\nbinlog-format=row\ngtid-mode=ON\nenforce_gtid_consistency=ON\n' | sudo tee /etc/mysql/conf.d/replication.cnf
19+
sudo service mysql start
20+
sudo mysql -h 127.0.0.1 -uroot -proot -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password'; FLUSH PRIVILEGES;"
21+
# create ssl/rsa files for mysql ssl support
22+
sudo mysql_ssl_rsa_setup --uid=mysql
23+
mysql -e "CREATE DATABASE IF NOT EXISTS test;" -uroot
24+
mysql -e "SHOW VARIABLES LIKE 'log_bin'" -uroot
25+
- name: Prepare for Go
26+
run: |
27+
sudo apt-get install -y make gcc
28+
- name: Install Go
29+
uses: actions/setup-go@v2
30+
with:
31+
go-version: ${{ matrix.go }}
32+
- name: Checkout code
33+
uses: actions/checkout@v1
34+
- name: Run tests
35+
run: go test ./...
36+
37+
golangci:
38+
name: golangci
39+
runs-on: ubuntu-latest
40+
steps:
41+
- uses: actions/checkout@v2
42+
- name: golangci-lint
43+
uses: golangci/golangci-lint-action@v2
44+
with:
45+
version: latest

‎.golangci.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
linters:
2+
disable-all: true
3+
enable:
4+
# All code is ready for:
5+
- deadcode
6+
- errcheck
7+
- staticcheck
8+
- structcheck
9+
- typecheck
10+
- unused
11+
- varcheck
12+
- misspell
13+
- nolintlint
14+
- goimports
15+
- nakedret
16+
- unconvert
17+
- whitespace
18+
# ToDo:
19+
#- gosimple
20+
#- govet
21+
#- ineffassign
22+
#- gocritic
23+
#- golint
24+
linters-settings:
25+
nolintlint:
26+
allow-unused: false
27+
allow-leading-space: false
28+
require-specific: true

‎.travis.yml

-47
This file was deleted.

‎canal/canal.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ func (c *Canal) Close() {
249249
c.conn = nil
250250
c.connLock.Unlock()
251251

252-
c.eventHandler.OnPosSynced(c.master.Position(), c.master.GTIDSet(), true)
252+
_ = c.eventHandler.OnPosSynced(c.master.Position(), c.master.GTIDSet(), true)
253253
}
254254

255255
func (c *Canal) WaitDumpDone() <-chan struct{} {

‎canal/canal_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,8 @@ func TestDropTableExp(t *testing.T) {
283283
}
284284
}
285285
}
286-
func TestWithoutSchemeExp(t *testing.T) {
287286

287+
func TestWithoutSchemeExp(t *testing.T) {
288288
cases := []replication.QueryEvent{
289289
replication.QueryEvent{
290290
Schema: []byte("test"),

‎canal/sync.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ func parseStmt(stmt ast.StmtNode) (ns []*node) {
227227
}
228228
ns = []*node{n}
229229
}
230-
return
230+
return ns
231231
}
232232

233233
func (c *Canal) updateTable(db, table string) (err error) {

‎client/auth.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func (c *Conn) readInitialHandshake() error {
4646
pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1
4747

4848
// connection id length is 4
49-
c.connectionID = uint32(binary.LittleEndian.Uint32(data[pos : pos+4]))
49+
c.connectionID = binary.LittleEndian.Uint32(data[pos : pos+4])
5050
pos += 4
5151

5252
c.salt = []byte{}
@@ -199,7 +199,7 @@ func (c *Conn) writeAuthHandshake() error {
199199

200200
// Charset [1 byte]
201201
// use default collation id 33 here, is utf-8
202-
data[12] = byte(DEFAULT_COLLATION_ID)
202+
data[12] = DEFAULT_COLLATION_ID
203203

204204
// SSL Connection Request Packet
205205
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest

‎client/client_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,6 @@ func (s *clientTestSuite) TestStmt_Select(c *C) {
289289

290290
e, _ = result.GetStringByName(0, "e")
291291
c.Assert(e, Equals, "test1")
292-
293292
}
294293

295294
func (s *clientTestSuite) TestStmt_NULL(c *C) {
@@ -309,11 +308,13 @@ func (s *clientTestSuite) TestStmt_NULL(c *C) {
309308

310309
str = `select * from mixer_test_stmt where id = ?`
311310
stmt, err = s.c.Prepare(str)
311+
c.Assert(err, IsNil)
312+
312313
defer stmt.Close()
313314

315+
result, err = stmt.Execute(2)
314316
c.Assert(err, IsNil)
315317

316-
result, err = stmt.Execute(2)
317318
b, err := result.IsNullByName(0, "id")
318319
c.Assert(err, IsNil)
319320
c.Assert(b, Equals, false)

‎client/resp.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ func (c *Conn) handleAuthResult() error {
101101
}
102102
c.authPluginName = switchToPlugin
103103
auth, addNull, err := c.genAuthResponse(data)
104+
if err != nil {
105+
return err
106+
}
107+
104108
if err = c.WriteAuthSwitchPacket(auth, addNull); err != nil {
105109
return err
106110
}
@@ -138,7 +142,7 @@ func (c *Conn) handleAuthResult() error {
138142
}
139143
}
140144
} else {
141-
errors.Errorf("invalid packet")
145+
return errors.Errorf("invalid packet %x", data[0])
142146
}
143147
} else if c.authPluginName == AUTH_SHA256_PASSWORD {
144148
if len(data) == 0 {
@@ -169,7 +173,6 @@ func (c *Conn) readAuthResult() ([]byte, string, error) {
169173
// see: https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/
170174
// packet indicator
171175
switch data[0] {
172-
173176
case OK_HEADER:
174177
_, err := c.handleOKPacket(data)
175178
return nil, "", err

‎client/stmt.go

+6-8
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@ import (
1010
)
1111

1212
type Stmt struct {
13-
conn *Conn
14-
id uint32
15-
query string
13+
conn *Conn
14+
id uint32
1615

1716
params int
1817
columns int
@@ -55,7 +54,7 @@ func (s *Stmt) write(args ...interface{}) error {
5554
//NULL-bitmap, length: (num-params+7)
5655
nullBitmap := make([]byte, (paramsNum+7)>>3)
5756

58-
var length int = int(1 + 4 + 1 + 4 + ((paramsNum + 7) >> 3) + 1 + (paramsNum << 1))
57+
length := 1 + 4 + 1 + 4 + ((paramsNum + 7) >> 3) + 1 + (paramsNum << 1)
5958

6059
var newParamBoundFlag byte = 0
6160

@@ -91,26 +90,25 @@ func (s *Stmt) write(args ...interface{}) error {
9190
case uint16:
9291
paramTypes[i<<1] = MYSQL_TYPE_SHORT
9392
paramTypes[(i<<1)+1] = 0x80
94-
paramValues[i] = Uint16ToBytes(uint16(v))
93+
paramValues[i] = Uint16ToBytes(v)
9594
case uint32:
9695
paramTypes[i<<1] = MYSQL_TYPE_LONG
9796
paramTypes[(i<<1)+1] = 0x80
98-
paramValues[i] = Uint32ToBytes(uint32(v))
97+
paramValues[i] = Uint32ToBytes(v)
9998
case uint:
10099
paramTypes[i<<1] = MYSQL_TYPE_LONGLONG
101100
paramTypes[(i<<1)+1] = 0x80
102101
paramValues[i] = Uint64ToBytes(uint64(v))
103102
case uint64:
104103
paramTypes[i<<1] = MYSQL_TYPE_LONGLONG
105104
paramTypes[(i<<1)+1] = 0x80
106-
paramValues[i] = Uint64ToBytes(uint64(v))
105+
paramValues[i] = Uint64ToBytes(v)
107106
case bool:
108107
paramTypes[i<<1] = MYSQL_TYPE_TINY
109108
if v {
110109
paramValues[i] = []byte{1}
111110
} else {
112111
paramValues[i] = []byte{0}
113-
114112
}
115113
case float32:
116114
paramTypes[i<<1] = MYSQL_TYPE_FLOAT

‎cmd/go-canal/main.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ var flavor = flag.String("flavor", "mysql", "Flavor: mysql or mariadb")
2323
var serverID = flag.Int("server-id", 101, "Unique Server ID")
2424
var mysqldump = flag.String("mysqldump", "mysqldump", "mysqldump execution path")
2525

26-
var dbs = flag.String("dbs", "test", "dump databases, seperated by comma")
27-
var tables = flag.String("tables", "", "dump tables, seperated by comma, will overwrite dbs")
26+
var dbs = flag.String("dbs", "test", "dump databases, separated by comma")
27+
var tables = flag.String("tables", "", "dump tables, separated by comma, will overwrite dbs")
2828
var tableDB = flag.String("table_db", "test", "database for dump tables")
2929
var ignoreTables = flag.String("ignore_tables", "", "ignore tables, must be database.table format, separated by comma")
3030

@@ -89,8 +89,7 @@ func main() {
8989

9090
sc := make(chan os.Signal, 1)
9191
signal.Notify(sc,
92-
os.Kill,
93-
os.Interrupt,
92+
syscall.SIGINT,
9493
syscall.SIGHUP,
9594
syscall.SIGINT,
9695
syscall.SIGTERM,

‎cmd/go-mysqlbinlog/main.go

-1
Original file line numberDiff line numberDiff line change
@@ -77,5 +77,4 @@ func main() {
7777
e.Dump(os.Stdout)
7878
}
7979
}
80-
8180
}

‎cmd/go-mysqldump/main.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ var password = flag.String("password", "", "MySQL password")
1616
var execution = flag.String("exec", "mysqldump", "mysqldump execution path")
1717
var output = flag.String("o", "", "dump output, empty for stdout")
1818

19-
var dbs = flag.String("dbs", "", "dump databases, seperated by comma")
20-
var tables = flag.String("tables", "", "dump tables, seperated by comma, will overwrite dbs")
19+
var dbs = flag.String("dbs", "", "dump databases, separated by comma")
20+
var tables = flag.String("tables", "", "dump tables, separated by comma, will overwrite dbs")
2121
var tableDB = flag.String("table_db", "", "database for dump tables")
2222
var ignoreTables = flag.String("ignore_tables", "", "ignore tables, must be database.table format, separated by comma")
2323

‎dump/dump.go

+30-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dump
22

33
import (
4+
"bytes"
45
"fmt"
56
"io"
67
"os"
@@ -40,6 +41,9 @@ type Dumper struct {
4041
masterDataSkipped bool
4142
maxAllowedPacket int
4243
hexBlob bool
44+
45+
// see detectColumnStatisticsParamSupported
46+
isColumnStatisticsParamSupported bool
4347
}
4448

4549
func NewDumper(executionPath string, addr string, user string, password string) (*Dumper, error) {
@@ -63,12 +67,28 @@ func NewDumper(executionPath string, addr string, user string, password string)
6367
d.IgnoreTables = make(map[string][]string)
6468
d.ExtraOptions = make([]string, 0, 5)
6569
d.masterDataSkipped = false
70+
d.isColumnStatisticsParamSupported = d.detectColumnStatisticsParamSupported()
6671

6772
d.ErrOut = os.Stderr
6873

6974
return d, nil
7075
}
7176

77+
// New mysqldump versions try to send queries to information_schema.COLUMN_STATISTICS table which does not exist in old MySQL (<5.x).
78+
// And we got error: "Unknown table 'COLUMN_STATISTICS' in information_schema (1109)".
79+
//
80+
// mysqldump may not send this query if it is started with parameter --column-statistics.
81+
// But this parameter exists only for versions >=8.0.2 (https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-2.html).
82+
//
83+
// For environments where the version of mysql-server and mysqldump differs, we try to check this parameter and use it if available.
84+
func (d *Dumper) detectColumnStatisticsParamSupported() bool {
85+
out, err := exec.Command(d.ExecutionPath, `--help`).CombinedOutput()
86+
if err != nil {
87+
return false
88+
}
89+
return bytes.Contains(out, []byte(`--column-statistics`))
90+
}
91+
7292
func (d *Dumper) SetCharset(charset string) {
7393
d.Charset = charset
7494
}
@@ -196,6 +216,10 @@ func (d *Dumper) Dump(w io.Writer) error {
196216
args = append(args, d.ExtraOptions...)
197217
}
198218

219+
if d.isColumnStatisticsParamSupported {
220+
args = append(args, `--column-statistics=0`)
221+
}
222+
199223
if len(d.Tables) == 0 && len(d.Databases) == 0 {
200224
args = append(args, "--all-databases")
201225
} else if len(d.Tables) == 0 {
@@ -208,7 +232,10 @@ func (d *Dumper) Dump(w io.Writer) error {
208232
// If we only dump some tables, the dump data will not have database name
209233
// which makes us hard to parse, so here we add it manually.
210234

211-
w.Write([]byte(fmt.Sprintf("USE `%s`;\n", d.TableDB)))
235+
_, err := w.Write([]byte(fmt.Sprintf("USE `%s`;\n", d.TableDB)))
236+
if err != nil {
237+
return fmt.Errorf(`could not write USE command: %w`, err)
238+
}
212239
}
213240

214241
log.Infof("exec mysqldump with %v", args)
@@ -227,12 +254,12 @@ func (d *Dumper) DumpAndParse(h ParseHandler) error {
227254
done := make(chan error, 1)
228255
go func() {
229256
err := Parse(r, h, !d.masterDataSkipped)
230-
r.CloseWithError(err)
257+
_ = r.CloseWithError(err)
231258
done <- err
232259
}()
233260

234261
err := d.Dump(w)
235-
w.CloseWithError(err)
262+
_ = w.CloseWithError(err)
236263

237264
err = <-done
238265

0 commit comments

Comments
 (0)