From b50c91af388a902a897630f81c00a94d5c6a6659 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 28 Jul 2018 22:00:15 +0800 Subject: [PATCH 1/4] use better log and update vendor --- Gopkg.lock | 31 +- Gopkg.toml | 6 - cmd/go-mysql-elasticsearch/main.go | 2 +- river/master.go | 2 +- river/river.go | 2 +- river/status.go | 2 +- river/sync.go | 2 +- vendor/github.com/BurntSushi/toml/COPYING | 14 - .../toml/cmd/toml-test-decoder/COPYING | 14 - .../toml/cmd/toml-test-encoder/COPYING | 14 - .../BurntSushi/toml/cmd/tomlv/COPYING | 14 - vendor/github.com/juju/errors/errortypes.go | 24 + .../shopspring/decimal/decimal-go.go | 414 ++++++++++++++++++ .../github.com/shopspring/decimal/decimal.go | 341 ++++++++++++++- .../github.com/shopspring/decimal/rounding.go | 118 +++++ .../siddontang/go-log/LICENSE} | 6 +- .../github.com/siddontang/go-log/log/doc.go | 20 + .../siddontang/go-log/log/filehandler.go | 230 ++++++++++ .../siddontang/go-log/log/handler.go | 54 +++ .../github.com/siddontang/go-log/log/log.go | 137 ++++++ .../siddontang/go-log/log/logger.go | 340 ++++++++++++++ .../siddontang/go-log/loggers}/loggers.go | 22 + .../siddontang/go-mysql/canal/canal.go | 8 +- .../siddontang/go-mysql/canal/dump.go | 33 +- .../siddontang/go-mysql/canal/master.go | 14 +- .../siddontang/go-mysql/canal/sync.go | 53 ++- .../siddontang/go-mysql/dump/dump.go | 3 - .../siddontang/go-mysql/mysql/resultset.go | 3 +- .../go-mysql/mysql/resultset_helper.go | 63 ++- .../go-mysql/replication/binlogstreamer.go | 2 +- .../go-mysql/replication/binlogsyncer.go | 8 +- .../siddontang/go-mysql/replication/parser.go | 21 +- .../go-mysql/replication/row_event.go | 32 +- vendor/gopkg.in/birkirb/loggers.v1/log/log.go | 141 ------ .../birkirb/loggers.v1/mappers/advanced.go | 77 ---- .../birkirb/loggers.v1/mappers/contextual.go | 33 -- .../birkirb/loggers.v1/mappers/mappers.go | 85 ---- .../birkirb/loggers.v1/mappers/standard.go | 63 --- .../loggers.v1/mappers/stdlib/stdlib.go | 123 ------ .../loggers.v1/mappers/stdlib/testing.go | 93 ---- 40 files changed, 1886 insertions(+), 778 deletions(-) delete mode 100644 vendor/github.com/BurntSushi/toml/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING create mode 100644 vendor/github.com/shopspring/decimal/decimal-go.go create mode 100644 vendor/github.com/shopspring/decimal/rounding.go rename vendor/{gopkg.in/birkirb/loggers.v1/LICENSE.txt => github.com/siddontang/go-log/LICENSE} (94%) create mode 100644 vendor/github.com/siddontang/go-log/log/doc.go create mode 100644 vendor/github.com/siddontang/go-log/log/filehandler.go create mode 100644 vendor/github.com/siddontang/go-log/log/handler.go create mode 100644 vendor/github.com/siddontang/go-log/log/log.go create mode 100644 vendor/github.com/siddontang/go-log/log/logger.go rename vendor/{gopkg.in/birkirb/loggers.v1 => github.com/siddontang/go-log/loggers}/loggers.go (51%) delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/log/log.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/advanced.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/contextual.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/mappers.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/standard.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/stdlib.go delete mode 100644 vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/testing.go diff --git a/Gopkg.lock b/Gopkg.lock index d57597bf..d8827cc9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,7 +11,7 @@ branch = "master" name = "github.com/juju/errors" packages = ["."] - revision = "c7d06af17c68cd34c835053720b21f6549d9b0ee" + revision = "812b06ada1776ad4dd95d575e18ffffe3a9ac34a" [[projects]] branch = "master" @@ -28,8 +28,8 @@ [[projects]] name = "github.com/shopspring/decimal" packages = ["."] - revision = "69b3a8ad1f5f2c8bd855cb6506d18593064a346b" - version = "1.0.1" + revision = "cd690d0c9e2447b1ef2a129a6b7b49077da89b8e" + version = "1.1.0" [[projects]] branch = "master" @@ -41,6 +41,15 @@ ] revision = "bdc77568d726a8702315ec4eafda030b6abc4f43" +[[projects]] + branch = "master" + name = "github.com/siddontang/go-log" + packages = [ + "log", + "loggers" + ] + revision = "a4d157e46fa3e08b7e7ff329af341fa3ff86c02c" + [[projects]] branch = "master" name = "github.com/siddontang/go-mysql" @@ -53,23 +62,11 @@ "replication", "schema" ] - revision = "fb8f61a50302b65f98d88cf0c5d02a4631fa6ec6" - -[[projects]] - branch = "master" - name = "gopkg.in/birkirb/loggers.v1" - packages = [ - ".", - "log", - "mappers", - "mappers/stdlib" - ] - revision = "fa51471f8169216876525e7b6c9fbd0c8789b57a" - source = "https://github.com/siddontang/loggers.git" + revision = "51eceecc377e5de21b7e5f43d3fb4218d305ac4a" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "f6ac894c2b53acb3d996d3324956a2f487ee253525e40aa05164c9e013f75e49" + inputs-digest = "a44edb92429a19dab5e22a895bbcd4f8d98e67fb9e5f0d6c6899afe1a6517b59" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c503b41d..7f50f77e 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -45,12 +45,6 @@ branch = "master" name = "github.com/siddontang/go-mysql" -[[constraint]] - name = "gopkg.in/birkirb/loggers.v1" - branch = "master" - source = "https://github.com/siddontang/loggers.git" - - [prune] go-tests = true unused-packages = true diff --git a/cmd/go-mysql-elasticsearch/main.go b/cmd/go-mysql-elasticsearch/main.go index 05739a6b..9c8856a9 100644 --- a/cmd/go-mysql-elasticsearch/main.go +++ b/cmd/go-mysql-elasticsearch/main.go @@ -8,8 +8,8 @@ import ( "syscall" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql-elasticsearch/river" - "gopkg.in/birkirb/loggers.v1/log" ) var configFile = flag.String("config", "./etc/river.toml", "go-mysql-elasticsearch config file") diff --git a/river/master.go b/river/master.go index d409d18c..2eb0afe8 100644 --- a/river/master.go +++ b/river/master.go @@ -9,9 +9,9 @@ import ( "github.com/BurntSushi/toml" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go/ioutil2" - "gopkg.in/birkirb/loggers.v1/log" ) type masterInfo struct { diff --git a/river/river.go b/river/river.go index 2ab8c0d1..1976d312 100644 --- a/river/river.go +++ b/river/river.go @@ -8,9 +8,9 @@ import ( "sync" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql-elasticsearch/elastic" "github.com/siddontang/go-mysql/canal" - "gopkg.in/birkirb/loggers.v1/log" ) // ErrRuleNotExist is the error if rule is not defined. diff --git a/river/status.go b/river/status.go index 52dd56f0..99e6a846 100644 --- a/river/status.go +++ b/river/status.go @@ -7,8 +7,8 @@ import ( "net/http" "net/http/pprof" + "github.com/siddontang/go-log/log" "github.com/siddontang/go/sync2" - "gopkg.in/birkirb/loggers.v1/log" ) type stat struct { diff --git a/river/sync.go b/river/sync.go index 5f09997f..4d842e62 100644 --- a/river/sync.go +++ b/river/sync.go @@ -9,12 +9,12 @@ import ( "time" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql-elasticsearch/elastic" "github.com/siddontang/go-mysql/canal" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go-mysql/schema" - "gopkg.in/birkirb/loggers.v1/log" ) const ( diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/juju/errors/errortypes.go b/vendor/github.com/juju/errors/errortypes.go index 9b731c44..5faf1e22 100644 --- a/vendor/github.com/juju/errors/errortypes.go +++ b/vendor/github.com/juju/errors/errortypes.go @@ -17,6 +17,30 @@ func wrap(err error, format, suffix string, args ...interface{}) Err { return newErr } +// timeout represents an error on timeout. +type timeout struct { + Err +} + +// Timeoutf returns an error which satisfies IsTimeout(). +func Timeoutf(format string, args ...interface{}) error { + return &timeout{wrap(nil, format, " timeout", args...)} +} + +// NewTimeout returns an error which wraps err that satisfies +// IsTimeout(). +func NewTimeout(err error, msg string) error { + return &timeout{wrap(err, msg, "")} +} + +// IsTimeout reports whether err was created with Timeoutf() or +// NewTimeout(). +func IsTimeout(err error) bool { + err = Cause(err) + _, ok := err.(*timeout) + return ok +} + // notFound represents an error when something has not been found. type notFound struct { Err diff --git a/vendor/github.com/shopspring/decimal/decimal-go.go b/vendor/github.com/shopspring/decimal/decimal-go.go new file mode 100644 index 00000000..e08a15ce --- /dev/null +++ b/vendor/github.com/shopspring/decimal/decimal-go.go @@ -0,0 +1,414 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. +package decimal + +type decimal struct { + d [800]byte // digits, big-endian representation + nd int // number of digits used + dp int // decimal point + neg bool // negative flag + trunc bool // discarded nonzero digits beyond d[:nd] +} + +func (a *decimal) String() string { + n := 10 + a.nd + if a.dp > 0 { + n += a.dp + } + if a.dp < 0 { + n += -a.dp + } + + buf := make([]byte, n) + w := 0 + switch { + case a.nd == 0: + return "0" + + case a.dp <= 0: + // zeros fill space between decimal point and digits + buf[w] = '0' + w++ + buf[w] = '.' + w++ + w += digitZero(buf[w : w+-a.dp]) + w += copy(buf[w:], a.d[0:a.nd]) + + case a.dp < a.nd: + // decimal point in middle of digits + w += copy(buf[w:], a.d[0:a.dp]) + buf[w] = '.' + w++ + w += copy(buf[w:], a.d[a.dp:a.nd]) + + default: + // zeros fill space between digits and decimal point + w += copy(buf[w:], a.d[0:a.nd]) + w += digitZero(buf[w : w+a.dp-a.nd]) + } + return string(buf[0:w]) +} + +func digitZero(dst []byte) int { + for i := range dst { + dst[i] = '0' + } + return len(dst) +} + +// trim trailing zeros from number. +// (They are meaningless; the decimal point is tracked +// independent of the number of digits.) +func trim(a *decimal) { + for a.nd > 0 && a.d[a.nd-1] == '0' { + a.nd-- + } + if a.nd == 0 { + a.dp = 0 + } +} + +// Assign v to a. +func (a *decimal) Assign(v uint64) { + var buf [24]byte + + // Write reversed decimal in buf. + n := 0 + for v > 0 { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n++ + v = v1 + } + + // Reverse again to produce forward decimal in a.d. + a.nd = 0 + for n--; n >= 0; n-- { + a.d[a.nd] = buf[n] + a.nd++ + } + a.dp = a.nd + trim(a) +} + +// Maximum shift that we can do in one pass without overflow. +// A uint has 32 or 64 bits, and we have to be able to accommodate 9<> 63) +const maxShift = uintSize - 4 + +// Binary shift right (/ 2) by k bits. k <= maxShift to avoid overflow. +func rightShift(a *decimal, k uint) { + r := 0 // read pointer + w := 0 // write pointer + + // Pick up enough leading digits to cover first shift. + var n uint + for ; n>>k == 0; r++ { + if r >= a.nd { + if n == 0 { + // a == 0; shouldn't get here, but handle anyway. + a.nd = 0 + return + } + for n>>k == 0 { + n = n * 10 + r++ + } + break + } + c := uint(a.d[r]) + n = n*10 + c - '0' + } + a.dp -= r - 1 + + var mask uint = (1 << k) - 1 + + // Pick up a digit, put down a digit. + for ; r < a.nd; r++ { + c := uint(a.d[r]) + dig := n >> k + n &= mask + a.d[w] = byte(dig + '0') + w++ + n = n*10 + c - '0' + } + + // Put down extra digits. + for n > 0 { + dig := n >> k + n &= mask + if w < len(a.d) { + a.d[w] = byte(dig + '0') + w++ + } else if dig > 0 { + a.trunc = true + } + n = n * 10 + } + + a.nd = w + trim(a) +} + +// Cheat sheet for left shift: table indexed by shift count giving +// number of new digits that will be introduced by that shift. +// +// For example, leftcheats[4] = {2, "625"}. That means that +// if we are shifting by 4 (multiplying by 16), it will add 2 digits +// when the string prefix is "625" through "999", and one fewer digit +// if the string prefix is "000" through "624". +// +// Credit for this trick goes to Ken. + +type leftCheat struct { + delta int // number of new digits + cutoff string // minus one digit if original < a. +} + +var leftcheats = []leftCheat{ + // Leading digits of 1/2^i = 5^i. + // 5^23 is not an exact 64-bit floating point number, + // so have to use bc for the math. + // Go up to 60 to be large enough for 32bit and 64bit platforms. + /* + seq 60 | sed 's/^/5^/' | bc | + awk 'BEGIN{ print "\t{ 0, \"\" }," } + { + log2 = log(2)/log(10) + printf("\t{ %d, \"%s\" },\t// * %d\n", + int(log2*NR+1), $0, 2**NR) + }' + */ + {0, ""}, + {1, "5"}, // * 2 + {1, "25"}, // * 4 + {1, "125"}, // * 8 + {2, "625"}, // * 16 + {2, "3125"}, // * 32 + {2, "15625"}, // * 64 + {3, "78125"}, // * 128 + {3, "390625"}, // * 256 + {3, "1953125"}, // * 512 + {4, "9765625"}, // * 1024 + {4, "48828125"}, // * 2048 + {4, "244140625"}, // * 4096 + {4, "1220703125"}, // * 8192 + {5, "6103515625"}, // * 16384 + {5, "30517578125"}, // * 32768 + {5, "152587890625"}, // * 65536 + {6, "762939453125"}, // * 131072 + {6, "3814697265625"}, // * 262144 + {6, "19073486328125"}, // * 524288 + {7, "95367431640625"}, // * 1048576 + {7, "476837158203125"}, // * 2097152 + {7, "2384185791015625"}, // * 4194304 + {7, "11920928955078125"}, // * 8388608 + {8, "59604644775390625"}, // * 16777216 + {8, "298023223876953125"}, // * 33554432 + {8, "1490116119384765625"}, // * 67108864 + {9, "7450580596923828125"}, // * 134217728 + {9, "37252902984619140625"}, // * 268435456 + {9, "186264514923095703125"}, // * 536870912 + {10, "931322574615478515625"}, // * 1073741824 + {10, "4656612873077392578125"}, // * 2147483648 + {10, "23283064365386962890625"}, // * 4294967296 + {10, "116415321826934814453125"}, // * 8589934592 + {11, "582076609134674072265625"}, // * 17179869184 + {11, "2910383045673370361328125"}, // * 34359738368 + {11, "14551915228366851806640625"}, // * 68719476736 + {12, "72759576141834259033203125"}, // * 137438953472 + {12, "363797880709171295166015625"}, // * 274877906944 + {12, "1818989403545856475830078125"}, // * 549755813888 + {13, "9094947017729282379150390625"}, // * 1099511627776 + {13, "45474735088646411895751953125"}, // * 2199023255552 + {13, "227373675443232059478759765625"}, // * 4398046511104 + {13, "1136868377216160297393798828125"}, // * 8796093022208 + {14, "5684341886080801486968994140625"}, // * 17592186044416 + {14, "28421709430404007434844970703125"}, // * 35184372088832 + {14, "142108547152020037174224853515625"}, // * 70368744177664 + {15, "710542735760100185871124267578125"}, // * 140737488355328 + {15, "3552713678800500929355621337890625"}, // * 281474976710656 + {15, "17763568394002504646778106689453125"}, // * 562949953421312 + {16, "88817841970012523233890533447265625"}, // * 1125899906842624 + {16, "444089209850062616169452667236328125"}, // * 2251799813685248 + {16, "2220446049250313080847263336181640625"}, // * 4503599627370496 + {16, "11102230246251565404236316680908203125"}, // * 9007199254740992 + {17, "55511151231257827021181583404541015625"}, // * 18014398509481984 + {17, "277555756156289135105907917022705078125"}, // * 36028797018963968 + {17, "1387778780781445675529539585113525390625"}, // * 72057594037927936 + {18, "6938893903907228377647697925567626953125"}, // * 144115188075855872 + {18, "34694469519536141888238489627838134765625"}, // * 288230376151711744 + {18, "173472347597680709441192448139190673828125"}, // * 576460752303423488 + {19, "867361737988403547205962240695953369140625"}, // * 1152921504606846976 +} + +// Is the leading prefix of b lexicographically less than s? +func prefixIsLessThan(b []byte, s string) bool { + for i := 0; i < len(s); i++ { + if i >= len(b) { + return true + } + if b[i] != s[i] { + return b[i] < s[i] + } + } + return false +} + +// Binary shift left (* 2) by k bits. k <= maxShift to avoid overflow. +func leftShift(a *decimal, k uint) { + delta := leftcheats[k].delta + if prefixIsLessThan(a.d[0:a.nd], leftcheats[k].cutoff) { + delta-- + } + + r := a.nd // read index + w := a.nd + delta // write index + + // Pick up a digit, put down a digit. + var n uint + for r--; r >= 0; r-- { + n += (uint(a.d[r]) - '0') << k + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + // Put down extra digits. + for n > 0 { + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + a.nd += delta + if a.nd >= len(a.d) { + a.nd = len(a.d) + } + a.dp += delta + trim(a) +} + +// Binary shift left (k > 0) or right (k < 0). +func (a *decimal) Shift(k int) { + switch { + case a.nd == 0: + // nothing to do: a == 0 + case k > 0: + for k > maxShift { + leftShift(a, maxShift) + k -= maxShift + } + leftShift(a, uint(k)) + case k < 0: + for k < -maxShift { + rightShift(a, maxShift) + k += maxShift + } + rightShift(a, uint(-k)) + } +} + +// If we chop a at nd digits, should we round up? +func shouldRoundUp(a *decimal, nd int) bool { + if nd < 0 || nd >= a.nd { + return false + } + if a.d[nd] == '5' && nd+1 == a.nd { // exactly halfway - round to even + // if we truncated, a little higher than what's recorded - always round up + if a.trunc { + return true + } + return nd > 0 && (a.d[nd-1]-'0')%2 != 0 + } + // not halfway - digit tells all + return a.d[nd] >= '5' +} + +// Round a to nd digits (or fewer). +// If nd is zero, it means we're rounding +// just to the left of the digits, as in +// 0.09 -> 0.1. +func (a *decimal) Round(nd int) { + if nd < 0 || nd >= a.nd { + return + } + if shouldRoundUp(a, nd) { + a.RoundUp(nd) + } else { + a.RoundDown(nd) + } +} + +// Round a down to nd digits (or fewer). +func (a *decimal) RoundDown(nd int) { + if nd < 0 || nd >= a.nd { + return + } + a.nd = nd + trim(a) +} + +// Round a up to nd digits (or fewer). +func (a *decimal) RoundUp(nd int) { + if nd < 0 || nd >= a.nd { + return + } + + // round up + for i := nd - 1; i >= 0; i-- { + c := a.d[i] + if c < '9' { // can stop after this digit + a.d[i]++ + a.nd = i + 1 + return + } + } + + // Number is all 9s. + // Change to single 1 with adjusted decimal point. + a.d[0] = '1' + a.nd = 1 + a.dp++ +} + +// Extract integer part, rounded appropriately. +// No guarantees about overflow. +func (a *decimal) RoundedInteger() uint64 { + if a.dp > 20 { + return 0xFFFFFFFFFFFFFFFF + } + var i int + n := uint64(0) + for i = 0; i < a.dp && i < a.nd; i++ { + n = n*10 + uint64(a.d[i]-'0') + } + for ; i < a.dp; i++ { + n *= 10 + } + if shouldRoundUp(a, a.dp) { + n++ + } + return n +} diff --git a/vendor/github.com/shopspring/decimal/decimal.go b/vendor/github.com/shopspring/decimal/decimal.go index 20aa6080..134ece2f 100644 --- a/vendor/github.com/shopspring/decimal/decimal.go +++ b/vendor/github.com/shopspring/decimal/decimal.go @@ -171,17 +171,84 @@ func RequireFromString(value string) Decimal { // NewFromFloat converts a float64 to Decimal. // -// Example: -// -// NewFromFloat(123.45678901234567).String() // output: "123.4567890123456" -// NewFromFloat(.00000000000000001).String() // output: "0.00000000000000001" +// The converted number will contain the number of significant digits that can be +// represented in a float with reliable roundtrip. +// This is typically 15 digits, but may be more in some cases. +// See https://www.exploringbinary.com/decimal-precision-of-binary-floating-point-numbers/ for more information. // -// NOTE: some float64 numbers can take up about 300 bytes of memory in decimal representation. -// Consider using NewFromFloatWithExponent if space is more important than precision. +// For slightly faster conversion, use NewFromFloatWithExponent where you can specify the precision in absolute terms. // // NOTE: this will panic on NaN, +/-inf func NewFromFloat(value float64) Decimal { - return NewFromFloatWithExponent(value, math.MinInt32) + if value == 0 { + return New(0, 0) + } + return newFromFloat(value, math.Float64bits(value), &float64info) +} + +// NewFromFloat converts a float32 to Decimal. +// +// The converted number will contain the number of significant digits that can be +// represented in a float with reliable roundtrip. +// This is typically 6-8 digits depending on the input. +// See https://www.exploringbinary.com/decimal-precision-of-binary-floating-point-numbers/ for more information. +// +// For slightly faster conversion, use NewFromFloatWithExponent where you can specify the precision in absolute terms. +// +// NOTE: this will panic on NaN, +/-inf +func NewFromFloat32(value float32) Decimal { + if value == 0 { + return New(0, 0) + } + // XOR is workaround for https://github.com/golang/go/issues/26285 + a := math.Float32bits(value) ^ 0x80808080 + return newFromFloat(float64(value), uint64(a)^0x80808080, &float32info) +} + +func newFromFloat(val float64, bits uint64, flt *floatInfo) Decimal { + if math.IsNaN(val) || math.IsInf(val, 0) { + panic(fmt.Sprintf("Cannot create a Decimal from %v", val)) + } + exp := int(bits>>flt.mantbits) & (1<>(flt.expbits+flt.mantbits) != 0 + + roundShortest(&d, mant, exp, flt) + // If less than 19 digits, we can do calculation in an int64. + if d.nd < 19 { + tmp := int64(0) + m := int64(1) + for i := d.nd - 1; i >= 0; i-- { + tmp += m * int64(d.d[i]-'0') + m *= 10 + } + if d.neg { + tmp *= -1 + } + return Decimal{value: big.NewInt(tmp), exp: int32(d.dp) - int32(d.nd)} + } + dValue := new(big.Int) + dValue, ok := dValue.SetString(string(d.d[:d.nd]), 10) + if ok { + return Decimal{value: dValue, exp: int32(d.dp) - int32(d.nd)} + } + + return NewFromFloatWithExponent(val, int32(d.dp)-int32(d.nd)) } // NewFromFloatWithExponent converts a float64 to Decimal, with an arbitrary @@ -376,6 +443,18 @@ func (d Decimal) Mul(d2 Decimal) Decimal { } } +// Shift shifts the decimal in base 10. +// It shifts left when shift is positive and right if shift is negative. +// In simpler terms, the given value for shift is added to the exponent +// of the decimal. +func (d Decimal) Shift(shift int32) Decimal { + d.ensureInitialized() + return Decimal{ + value: new(big.Int).Set(d.value), + exp: d.exp + shift, + } +} + // Div returns d / d2. If it doesn't divide exactly, the result will have // DivisionPrecision digits after the decimal point. func (d Decimal) Div(d2 Decimal) Decimal { @@ -544,6 +623,33 @@ func (d Decimal) Sign() int { return d.value.Sign() } +// IsPositive return +// +// true if d > 0 +// false if d == 0 +// false if d < 0 +func (d Decimal) IsPositive() bool { + return d.Sign() == 1 +} + +// IsNegative return +// +// true if d < 0 +// false if d == 0 +// false if d > 0 +func (d Decimal) IsNegative() bool { + return d.Sign() == -1 +} + +// IsZero return +// +// true if d == 0 +// false if d > 0 +// false if d < 0 +func (d Decimal) IsZero() bool { + return d.Sign() == 0 +} + // Exponent returns the exponent, or scale component of the decimal. func (d Decimal) Exponent() int32 { return d.exp @@ -1105,3 +1211,224 @@ func (d NullDecimal) MarshalJSON() ([]byte, error) { } return d.Decimal.MarshalJSON() } + +// Trig functions + +// Atan returns the arctangent, in radians, of x. +func (x Decimal) Atan() Decimal { + if x.Equal(NewFromFloat(0.0)) { + return x + } + if x.GreaterThan(NewFromFloat(0.0)) { + return x.satan() + } + return x.Neg().satan().Neg() +} + +func (d Decimal) xatan() Decimal { + P0 := NewFromFloat(-8.750608600031904122785e-01) + P1 := NewFromFloat(-1.615753718733365076637e+01) + P2 := NewFromFloat(-7.500855792314704667340e+01) + P3 := NewFromFloat(-1.228866684490136173410e+02) + P4 := NewFromFloat(-6.485021904942025371773e+01) + Q0 := NewFromFloat(2.485846490142306297962e+01) + Q1 := NewFromFloat(1.650270098316988542046e+02) + Q2 := NewFromFloat(4.328810604912902668951e+02) + Q3 := NewFromFloat(4.853903996359136964868e+02) + Q4 := NewFromFloat(1.945506571482613964425e+02) + z := d.Mul(d) + b1 := P0.Mul(z).Add(P1).Mul(z).Add(P2).Mul(z).Add(P3).Mul(z).Add(P4).Mul(z) + b2 := z.Add(Q0).Mul(z).Add(Q1).Mul(z).Add(Q2).Mul(z).Add(Q3).Mul(z).Add(Q4) + z = b1.Div(b2) + z = d.Mul(z).Add(d) + return z +} + +// satan reduces its argument (known to be positive) +// to the range [0, 0.66] and calls xatan. +func (d Decimal) satan() Decimal { + Morebits := NewFromFloat(6.123233995736765886130e-17) // pi/2 = PIO2 + Morebits + Tan3pio8 := NewFromFloat(2.41421356237309504880) // tan(3*pi/8) + pi := NewFromFloat(3.14159265358979323846264338327950288419716939937510582097494459) + + if d.LessThanOrEqual(NewFromFloat(0.66)) { + return d.xatan() + } + if d.GreaterThan(Tan3pio8) { + return pi.Div(NewFromFloat(2.0)).Sub(NewFromFloat(1.0).Div(d).xatan()).Add(Morebits) + } + return pi.Div(NewFromFloat(4.0)).Add((d.Sub(NewFromFloat(1.0)).Div(d.Add(NewFromFloat(1.0)))).xatan()).Add(NewFromFloat(0.5).Mul(Morebits)) +} + +// sin coefficients + var _sin = [...]Decimal{ + NewFromFloat(1.58962301576546568060E-10), // 0x3de5d8fd1fd19ccd + NewFromFloat(-2.50507477628578072866E-8), // 0xbe5ae5e5a9291f5d + NewFromFloat(2.75573136213857245213E-6), // 0x3ec71de3567d48a1 + NewFromFloat(-1.98412698295895385996E-4), // 0xbf2a01a019bfdf03 + NewFromFloat(8.33333333332211858878E-3), // 0x3f8111111110f7d0 + NewFromFloat(-1.66666666666666307295E-1), // 0xbfc5555555555548 + } + +// Sin returns the sine of the radian argument x. + func (d Decimal) Sin() Decimal { + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + if d.Equal(NewFromFloat(0.0)) { + return d + } + // make argument positive but save the sign + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + sign = true + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + j &= 7 // octant modulo 2Pi radians (360 degrees) + // reflect in x axis + if j > 3 { + sign = !sign + j -= 4 + } + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if j == 1 || j == 2 { + w := zz.Mul(zz).Mul(_cos[0].Mul(zz).Add(_cos[1]).Mul(zz).Add(_cos[2]).Mul(zz).Add(_cos[3]).Mul(zz).Add(_cos[4]).Mul(zz).Add(_cos[5])) + y = NewFromFloat(1.0).Sub(NewFromFloat(0.5).Mul(zz)).Add(w) + } else { + y = z.Add(z.Mul(zz).Mul(_sin[0].Mul(zz).Add(_sin[1]).Mul(zz).Add(_sin[2]).Mul(zz).Add(_sin[3]).Mul(zz).Add(_sin[4]).Mul(zz).Add(_sin[5]))) + } + if sign { + y = y.Neg() + } + return y + } + + // cos coefficients + var _cos = [...]Decimal{ + NewFromFloat(-1.13585365213876817300E-11), // 0xbda8fa49a0861a9b + NewFromFloat(2.08757008419747316778E-9), // 0x3e21ee9d7b4e3f05 + NewFromFloat(-2.75573141792967388112E-7), // 0xbe927e4f7eac4bc6 + NewFromFloat(2.48015872888517045348E-5), // 0x3efa01a019c844f5 + NewFromFloat(-1.38888888888730564116E-3), // 0xbf56c16c16c14f91 + NewFromFloat(4.16666666666665929218E-2), // 0x3fa555555555554b + } + + // Cos returns the cosine of the radian argument x. + func (d Decimal) Cos() Decimal { + + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + // make argument positive + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + j &= 7 // octant modulo 2Pi radians (360 degrees) + // reflect in x axis + if j > 3 { + sign = !sign + j -= 4 + } + if j > 1 { + sign = !sign + } + + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if j == 1 || j == 2 { + y = z.Add(z.Mul(zz).Mul(_sin[0].Mul(zz).Add(_sin[1]).Mul(zz).Add(_sin[2]).Mul(zz).Add(_sin[3]).Mul(zz).Add(_sin[4]).Mul(zz).Add(_sin[5]))) + } else { + w := zz.Mul(zz).Mul(_cos[0].Mul(zz).Add(_cos[1]).Mul(zz).Add(_cos[2]).Mul(zz).Add(_cos[3]).Mul(zz).Add(_cos[4]).Mul(zz).Add(_cos[5])) + y = NewFromFloat(1.0).Sub(NewFromFloat(0.5).Mul(zz)).Add(w) + } + if sign { + y = y.Neg() + } + return y + } + + var _tanP = [...]Decimal{ + NewFromFloat(-1.30936939181383777646E+4), // 0xc0c992d8d24f3f38 + NewFromFloat(1.15351664838587416140E+6), // 0x413199eca5fc9ddd + NewFromFloat(-1.79565251976484877988E+7), // 0xc1711fead3299176 + } + var _tanQ = [...]Decimal{ + NewFromFloat(1.00000000000000000000E+0), + NewFromFloat(1.36812963470692954678E+4), //0x40cab8a5eeb36572 + NewFromFloat(-1.32089234440210967447E+6), //0xc13427bc582abc96 + NewFromFloat(2.50083801823357915839E+7), //0x4177d98fc2ead8ef + NewFromFloat(-5.38695755929454629881E+7), //0xc189afe03cbe5a31 + } + + // Tan returns the tangent of the radian argument x. + func (d Decimal) Tan() Decimal { + + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + if d.Equal(NewFromFloat(0.0)) { + return d + } + + // make argument positive but save the sign + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + sign = true + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if zz.GreaterThan(NewFromFloat(1e-14)) { + w := zz.Mul(_tanP[0].Mul(zz).Add(_tanP[1]).Mul(zz).Add(_tanP[2])) + x := zz.Add(_tanQ[1]).Mul(zz).Add(_tanQ[2]).Mul(zz).Add(_tanQ[3]).Mul(zz).Add(_tanQ[4]) + y = z.Add(z.Mul(w.Div(x))) + } else { + y = z + } + if j&2 == 2 { + y = NewFromFloat(-1.0).Div(y) + } + if sign { + y = y.Neg() + } + return y + } diff --git a/vendor/github.com/shopspring/decimal/rounding.go b/vendor/github.com/shopspring/decimal/rounding.go new file mode 100644 index 00000000..fdd74eaa --- /dev/null +++ b/vendor/github.com/shopspring/decimal/rounding.go @@ -0,0 +1,118 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. +package decimal + +type floatInfo struct { + mantbits uint + expbits uint + bias int +} + +var float32info = floatInfo{23, 8, -127} +var float64info = floatInfo{52, 11, -1023} + +// roundShortest rounds d (= mant * 2^exp) to the shortest number of digits +// that will let the original floating point value be precisely reconstructed. +func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { + // If mantissa is zero, the number is zero; stop now. + if mant == 0 { + d.nd = 0 + return + } + + // Compute upper and lower such that any decimal number + // between upper and lower (possibly inclusive) + // will round to the original floating point number. + + // We may see at once that the number is already shortest. + // + // Suppose d is not denormal, so that 2^exp <= d < 10^dp. + // The closest shorter number is at least 10^(dp-nd) away. + // The lower/upper bounds computed below are at distance + // at most 2^(exp-mantbits). + // + // So the number is already shortest if 10^(dp-nd) > 2^(exp-mantbits), + // or equivalently log2(10)*(dp-nd) > exp-mantbits. + // It is true if 332/100*(dp-nd) >= exp-mantbits (log2(10) > 3.32). + minexp := flt.bias + 1 // minimum possible exponent + if exp > minexp && 332*(d.dp-d.nd) >= 100*(exp-int(flt.mantbits)) { + // The number is already shortest. + return + } + + // d = mant << (exp - mantbits) + // Next highest floating point number is mant+1 << exp-mantbits. + // Our upper bound is halfway between, mant*2+1 << exp-mantbits-1. + upper := new(decimal) + upper.Assign(mant*2 + 1) + upper.Shift(exp - int(flt.mantbits) - 1) + + // d = mant << (exp - mantbits) + // Next lowest floating point number is mant-1 << exp-mantbits, + // unless mant-1 drops the significant bit and exp is not the minimum exp, + // in which case the next lowest is mant*2-1 << exp-mantbits-1. + // Either way, call it mantlo << explo-mantbits. + // Our lower bound is halfway between, mantlo*2+1 << explo-mantbits-1. + var mantlo uint64 + var explo int + if mant > 1< 0 { + h.fd.Close() + + for i := h.backupCount - 1; i > 0; i-- { + sfn := fmt.Sprintf("%s.%d", h.fileName, i) + dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) + + os.Rename(sfn, dfn) + } + + dfn := fmt.Sprintf("%s.1", h.fileName) + os.Rename(h.fileName, dfn) + + h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + h.curBytes = 0 + f, err := h.fd.Stat() + if err != nil { + return + } + h.curBytes = int(f.Size()) + } +} + +// TimeRotatingFileHandler writes log to a file, +// it will backup current and open a new one, with a period time you sepecified. +// +// refer: http://docs.python.org/2/library/logging.handlers.html. +// same like python TimedRotatingFileHandler. +type TimeRotatingFileHandler struct { + fd *os.File + + baseName string + interval int64 + suffix string + rolloverAt int64 +} + +// TimeRotating way +const ( + WhenSecond = iota + WhenMinute + WhenHour + WhenDay +) + +// NewTimeRotatingFileHandler creates a TimeRotatingFileHandler +func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { + dir := path.Dir(baseName) + os.MkdirAll(dir, 0777) + + h := new(TimeRotatingFileHandler) + + h.baseName = baseName + + switch when { + case WhenSecond: + h.interval = 1 + h.suffix = "2006-01-02_15-04-05" + case WhenMinute: + h.interval = 60 + h.suffix = "2006-01-02_15-04" + case WhenHour: + h.interval = 3600 + h.suffix = "2006-01-02_15" + case WhenDay: + h.interval = 3600 * 24 + h.suffix = "2006-01-02" + default: + return nil, fmt.Errorf("invalid when_rotate: %d", when) + } + + h.interval = h.interval * int64(interval) + + var err error + h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + fInfo, _ := h.fd.Stat() + h.rolloverAt = fInfo.ModTime().Unix() + h.interval + + return h, nil +} + +func (h *TimeRotatingFileHandler) doRollover() { + //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py + now := time.Now() + + if h.rolloverAt <= now.Unix() { + fName := h.baseName + now.Format(h.suffix) + h.fd.Close() + e := os.Rename(h.baseName, fName) + if e != nil { + panic(e) + } + + h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + h.rolloverAt = time.Now().Unix() + h.interval + } +} + +// Write implements Handler interface +func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(b) +} + +// Close implements Handler interface +func (h *TimeRotatingFileHandler) Close() error { + return h.fd.Close() +} diff --git a/vendor/github.com/siddontang/go-log/log/handler.go b/vendor/github.com/siddontang/go-log/log/handler.go new file mode 100644 index 00000000..5460f069 --- /dev/null +++ b/vendor/github.com/siddontang/go-log/log/handler.go @@ -0,0 +1,54 @@ +package log + +import ( + "io" +) + +//Handler writes logs to somewhere +type Handler interface { + Write(p []byte) (n int, err error) + Close() error +} + +// StreamHandler writes logs to a specified io Writer, maybe stdout, stderr, etc... +type StreamHandler struct { + w io.Writer +} + +// NewStreamHandler creates a StreamHandler +func NewStreamHandler(w io.Writer) (*StreamHandler, error) { + h := new(StreamHandler) + + h.w = w + + return h, nil +} + +// Write implements Handler interface +func (h *StreamHandler) Write(b []byte) (n int, err error) { + return h.w.Write(b) +} + +// Close implements Handler interface +func (h *StreamHandler) Close() error { + return nil +} + +// NullHandler does nothing, it discards anything. +type NullHandler struct { +} + +// NewNullHandler creates a NullHandler +func NewNullHandler() (*NullHandler, error) { + return new(NullHandler), nil +} + +// // Write implements Handler interface +func (h *NullHandler) Write(b []byte) (n int, err error) { + return len(b), nil +} + +// Close implements Handler interface +func (h *NullHandler) Close() { + +} diff --git a/vendor/github.com/siddontang/go-log/log/log.go b/vendor/github.com/siddontang/go-log/log/log.go new file mode 100644 index 00000000..956186d9 --- /dev/null +++ b/vendor/github.com/siddontang/go-log/log/log.go @@ -0,0 +1,137 @@ +package log + +import ( + "fmt" + "os" +) + +var logger = NewDefault(newStdHandler()) + +// SetDefaultLogger changes the global logger +func SetDefaultLogger(l *Logger) { + logger = l +} + +// SetLevel changes the logger level +func SetLevel(level Level) { + logger.SetLevel(level) +} + +// SetLevelByName changes the logger level by name +func SetLevelByName(name string) { + logger.SetLevelByName(name) +} + +// Fatal records the log with fatal level and exits +func Fatal(args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprint(args...)) + os.Exit(1) +} + +// Fatalf records the log with fatal level and exits +func Fatalf(format string, args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +// Fatalln records the log with fatal level and exits +func Fatalln(args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprintln(args...)) + os.Exit(1) +} + +// Panic records the log with fatal level and panics +func Panic(args ...interface{}) { + msg := fmt.Sprint(args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Panicf records the log with fatal level and panics +func Panicf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Panicln records the log with fatal level and panics +func Panicln(args ...interface{}) { + msg := fmt.Sprintln(args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Print records the log with trace level +func Print(args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprint(args...)) +} + +// Printf records the log with trace level +func Printf(format string, args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprintf(format, args...)) +} + +// Println records the log with trace level +func Println(args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprintln(args...)) +} + +// Debug records the log with debug level +func Debug(args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprint(args...)) +} + +// Debugf records the log with debug level +func Debugf(format string, args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprintf(format, args...)) +} + +// Debugln records the log with debug level +func Debugln(args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprintln(args...)) +} + +// Error records the log with error level +func Error(args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprint(args...)) +} + +// Errorf records the log with error level +func Errorf(format string, args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprintf(format, args...)) +} + +// Errorln records the log with error level +func Errorln(args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprintln(args...)) +} + +// Info records the log with info level +func Info(args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprint(args...)) +} + +// Infof records the log with info level +func Infof(format string, args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprintf(format, args...)) +} + +// Infoln records the log with info level +func Infoln(args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprintln(args...)) +} + +// Warn records the log with warn level +func Warn(args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprint(args...)) +} + +// Warnf records the log with warn level +func Warnf(format string, args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprintf(format, args...)) +} + +// Warnln records the log with warn level +func Warnln(args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprintln(args...)) +} diff --git a/vendor/github.com/siddontang/go-log/log/logger.go b/vendor/github.com/siddontang/go-log/log/logger.go new file mode 100644 index 00000000..b2f7ed28 --- /dev/null +++ b/vendor/github.com/siddontang/go-log/log/logger.go @@ -0,0 +1,340 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/siddontang/go-log/loggers" +) + +const ( + timeFormat = "2006/01/02 15:04:05" + maxBufPoolSize = 16 +) + +// Logger flag +const ( + Ltime = 1 << iota // time format "2006/01/02 15:04:05" + Lfile // file.go:123 + Llevel // [Trace|Debug|Info...] +) + +// Level type +type Level int + +// Log level, from low to high, more high means more serious +const ( + LevelTrace Level = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelFatal +) + +// String returns level String +func (l Level) String() string { + switch l { + case LevelTrace: + return "trace" + case LevelDebug: + return "debug" + case LevelInfo: + return "info" + case LevelWarn: + return "warn" + case LevelError: + return "error" + case LevelFatal: + return "fatal" + } + // return default info + return "info" +} + +// Logger is the logger to record log +type Logger struct { + // TODO: support logger.Contextual + loggers.Advanced + + sync.Mutex + + level Level + flag int + + handler Handler + + quit chan struct{} + msg chan []byte + + bufs [][]byte +} + +// New creates a logger with specified handler and flag +func New(handler Handler, flag int) *Logger { + var l = new(Logger) + + l.level = LevelInfo + l.handler = handler + + l.flag = flag + + l.quit = make(chan struct{}) + + l.msg = make(chan []byte, 1024) + + l.bufs = make([][]byte, 0, 16) + + go l.run() + + return l +} + +// NewDefault creates default logger with specified handler and flag: Ltime|Lfile|Llevel +func NewDefault(handler Handler) *Logger { + return New(handler, Ltime|Lfile|Llevel) +} + +func newStdHandler() *StreamHandler { + h, _ := NewStreamHandler(os.Stdout) + return h +} + +func (l *Logger) run() { + for { + select { + case msg := <-l.msg: + l.handler.Write(msg) + l.putBuf(msg) + case <-l.quit: + l.handler.Close() + } + } +} + +func (l *Logger) popBuf() []byte { + l.Lock() + var buf []byte + if len(l.bufs) == 0 { + buf = make([]byte, 0, 1024) + } else { + buf = l.bufs[len(l.bufs)-1] + l.bufs = l.bufs[0 : len(l.bufs)-1] + } + l.Unlock() + + return buf +} + +func (l *Logger) putBuf(buf []byte) { + l.Lock() + if len(l.bufs) < maxBufPoolSize { + buf = buf[0:0] + l.bufs = append(l.bufs, buf) + } + l.Unlock() +} + +// Close closes the logger +func (l *Logger) Close() { + if l.quit == nil { + return + } + + close(l.quit) + l.quit = nil +} + +// SetLevel sets log level, any log level less than it will not log +func (l *Logger) SetLevel(level Level) { + l.level = level +} + +// SetLevelByName sets log level by name +func (l *Logger) SetLevelByName(name string) { + level := LevelInfo + switch strings.ToLower(name) { + case "trace": + level = LevelTrace + case "debug": + level = LevelDebug + case "warn", "warning": + level = LevelWarn + case "error": + level = LevelError + case "fatal": + level = LevelFatal + default: + level = LevelInfo + } + + l.SetLevel(level) +} + +// Output records the log with special callstack depth and log level. +func (l *Logger) Output(callDepth int, level Level, msg string) { + if l.level > level { + return + } + + buf := l.popBuf() + + if l.flag&Ltime > 0 { + now := time.Now().Format(timeFormat) + buf = append(buf, '[') + buf = append(buf, now...) + buf = append(buf, "] "...) + } + + if l.flag&Llevel > 0 { + buf = append(buf, '[') + buf = append(buf, level.String()...) + buf = append(buf, "] "...) + } + + if l.flag&Lfile > 0 { + _, file, line, ok := runtime.Caller(callDepth) + if !ok { + file = "???" + line = 0 + } else { + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + file = file[i+1:] + break + } + } + } + + buf = append(buf, file...) + buf = append(buf, ':') + + buf = strconv.AppendInt(buf, int64(line), 10) + buf = append(buf, ' ') + } + + buf = append(buf, msg...) + if len(msg) == 0 || msg[len(msg)-1] != '\n' { + buf = append(buf, '\n') + } + l.msg <- buf +} + +// Fatal records the log with fatal level and exits +func (l *Logger) Fatal(args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprint(args...)) + os.Exit(1) +} + +// Fatalf records the log with fatal level and exits +func (l *Logger) Fatalf(format string, args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +// Fatalln records the log with fatal level and exits +func (l *Logger) Fatalln(args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprintln(args...)) + os.Exit(1) +} + +// Panic records the log with fatal level and panics +func (l *Logger) Panic(args ...interface{}) { + msg := fmt.Sprint(args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Panicf records the log with fatal level and panics +func (l *Logger) Panicf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Panicln records the log with fatal level and panics +func (l *Logger) Panicln(args ...interface{}) { + msg := fmt.Sprintln(args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Print records the log with trace level +func (l *Logger) Print(args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprint(args...)) +} + +// Printf records the log with trace level +func (l *Logger) Printf(format string, args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprintf(format, args...)) +} + +// Println records the log with trace level +func (l *Logger) Println(args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprintln(args...)) +} + +// Debug records the log with debug level +func (l *Logger) Debug(args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprint(args...)) +} + +// Debugf records the log with debug level +func (l *Logger) Debugf(format string, args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprintf(format, args...)) +} + +// Debugln records the log with debug level +func (l *Logger) Debugln(args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprintln(args...)) +} + +// Error records the log with error level +func (l *Logger) Error(args ...interface{}) { + l.Output(2, LevelError, fmt.Sprint(args...)) +} + +// Errorf records the log with error level +func (l *Logger) Errorf(format string, args ...interface{}) { + l.Output(2, LevelError, fmt.Sprintf(format, args...)) +} + +// Errorln records the log with error level +func (l *Logger) Errorln(args ...interface{}) { + l.Output(2, LevelError, fmt.Sprintln(args...)) +} + +// Info records the log with info level +func (l *Logger) Info(args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprint(args...)) +} + +// Infof records the log with info level +func (l *Logger) Infof(format string, args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprintf(format, args...)) +} + +// Infoln records the log with info level +func (l *Logger) Infoln(args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprintln(args...)) +} + +// Warn records the log with warn level +func (l *Logger) Warn(args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprint(args...)) +} + +// Warnf records the log with warn level +func (l *Logger) Warnf(format string, args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprintf(format, args...)) +} + +// Warnln records the log with warn level +func (l *Logger) Warnln(args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprintln(args...)) +} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/loggers.go b/vendor/github.com/siddontang/go-log/loggers/loggers.go similarity index 51% rename from vendor/gopkg.in/birkirb/loggers.v1/loggers.go rename to vendor/github.com/siddontang/go-log/loggers/loggers.go index bd1b4e8f..2723b24a 100644 --- a/vendor/gopkg.in/birkirb/loggers.v1/loggers.go +++ b/vendor/github.com/siddontang/go-log/loggers/loggers.go @@ -1,3 +1,25 @@ +// MIT License + +// Copyright (c) 2017 Birkir A. Barkarson + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + package loggers // Standard is the interface used by Go's standard library's log package. diff --git a/vendor/github.com/siddontang/go-mysql/canal/canal.go b/vendor/github.com/siddontang/go-mysql/canal/canal.go index 6606c417..2f86adc6 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/canal.go +++ b/vendor/github.com/siddontang/go-mysql/canal/canal.go @@ -12,12 +12,12 @@ import ( "time" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/client" "github.com/siddontang/go-mysql/dump" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go-mysql/schema" - "gopkg.in/birkirb/loggers.v1/log" ) // Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc... @@ -178,7 +178,7 @@ func (c *Canal) RunFrom(pos mysql.Position) error { } func (c *Canal) StartFromGTID(set mysql.GTIDSet) error { - c.master.UpdateGTID(set) + c.master.UpdateGTIDSet(set) return c.Run() } @@ -455,6 +455,6 @@ func (c *Canal) SyncedPosition() mysql.Position { return c.master.Position() } -func (c *Canal) SyncedGTID() mysql.GTIDSet { - return c.master.GTID() +func (c *Canal) SyncedGTIDSet() mysql.GTIDSet { + return c.master.GTIDSet() } diff --git a/vendor/github.com/siddontang/go-mysql/canal/dump.go b/vendor/github.com/siddontang/go-mysql/canal/dump.go index 0d086f0c..212cdaac 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/dump.go +++ b/vendor/github.com/siddontang/go-mysql/canal/dump.go @@ -1,20 +1,22 @@ package canal import ( + "fmt" "strconv" "time" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/dump" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/schema" - "gopkg.in/birkirb/loggers.v1/log" ) type dumpParseHandler struct { c *Canal name string pos uint64 + gset mysql.GTIDSet } func (h *dumpParseHandler) BinLog(name string, pos uint64) error { @@ -99,10 +101,21 @@ func (c *Canal) AddDumpIgnoreTables(db string, tables ...string) { func (c *Canal) dump() error { if c.dumper == nil { - return errors.New("mysqldump is not exist") + return errors.New("mysqldump does not exist") } h := &dumpParseHandler{c: c} + // If users call StartFromGTID with empty position to start dumping with gtid, + // we record the current gtid position before dump starts. + // + // See tryDump() to see when dump is skipped. + if c.master.GTIDSet() != nil { + gset, err := c.GetMasterGTIDSet() + if err != nil { + return errors.Trace(err) + } + h.gset = gset + } if c.cfg.Dump.SkipMasterData { pos, err := c.GetMasterPos() @@ -120,22 +133,26 @@ func (c *Canal) dump() error { return errors.Trace(err) } - log.Infof("dump MySQL and parse OK, use %0.2f seconds, start binlog replication at (%s, %d)", - time.Now().Sub(start).Seconds(), h.name, h.pos) - pos := mysql.Position{h.name, uint32(h.pos)} c.master.Update(pos) c.eventHandler.OnPosSynced(pos, true) + var startPos fmt.Stringer = pos + if h.gset != nil { + c.master.UpdateGTIDSet(h.gset) + startPos = h.gset + } + log.Infof("dump MySQL and parse OK, use %0.2f seconds, start binlog replication at %s", + time.Now().Sub(start).Seconds(), startPos) return nil } func (c *Canal) tryDump() error { pos := c.master.Position() - gtid := c.master.GTID() + gset := c.master.GTIDSet() if (len(pos.Name) > 0 && pos.Pos > 0) || - (gtid != nil && gtid.String() != "") { + (gset != nil && gset.String() != "") { // we will sync with binlog name and position - log.Infof("skip dump, use last binlog replication pos %s or GTID %s", pos, gtid) + log.Infof("skip dump, use last binlog replication pos %s or GTID set %s", pos, gset) return nil } diff --git a/vendor/github.com/siddontang/go-mysql/canal/master.go b/vendor/github.com/siddontang/go-mysql/canal/master.go index 5d7a7e34..62bb4b77 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/master.go +++ b/vendor/github.com/siddontang/go-mysql/canal/master.go @@ -3,8 +3,8 @@ package canal import ( "sync" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" - "gopkg.in/birkirb/loggers.v1/log" ) type masterInfo struct { @@ -12,7 +12,7 @@ type masterInfo struct { pos mysql.Position - gtid mysql.GTIDSet + gset mysql.GTIDSet } func (m *masterInfo) Update(pos mysql.Position) { @@ -23,11 +23,11 @@ func (m *masterInfo) Update(pos mysql.Position) { m.Unlock() } -func (m *masterInfo) UpdateGTID(gtid mysql.GTIDSet) { - log.Debugf("update master gtid %s", gtid.String()) +func (m *masterInfo) UpdateGTIDSet(gset mysql.GTIDSet) { + log.Debugf("update master gtid set %s", gset) m.Lock() - m.gtid = gtid + m.gset = gset m.Unlock() } @@ -38,9 +38,9 @@ func (m *masterInfo) Position() mysql.Position { return m.pos } -func (m *masterInfo) GTID() mysql.GTIDSet { +func (m *masterInfo) GTIDSet() mysql.GTIDSet { m.RLock() defer m.RUnlock() - return m.gtid + return m.gset } diff --git a/vendor/github.com/siddontang/go-mysql/canal/sync.go b/vendor/github.com/siddontang/go-mysql/canal/sync.go index 38d0b107..77a8508d 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/sync.go +++ b/vendor/github.com/siddontang/go-mysql/canal/sync.go @@ -7,10 +7,10 @@ import ( "github.com/juju/errors" "github.com/satori/go.uuid" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go-mysql/schema" - "gopkg.in/birkirb/loggers.v1/log" ) var ( @@ -21,8 +21,8 @@ var ( ) func (c *Canal) startSyncer() (*replication.BinlogStreamer, error) { - gtid := c.master.GTID() - if gtid == nil || gtid.String() == "" { + gset := c.master.GTIDSet() + if gset == nil { pos := c.master.Position() s, err := c.syncer.StartSync(pos) if err != nil { @@ -31,11 +31,11 @@ func (c *Canal) startSyncer() (*replication.BinlogStreamer, error) { log.Infof("start sync binlog at binlog file %v", pos) return s, nil } else { - s, err := c.syncer.StartSyncGTID(gtid) + s, err := c.syncer.StartSyncGTID(gset) if err != nil { - return nil, errors.Errorf("start sync replication at GTID %v error %v", gtid, err) + return nil, errors.Errorf("start sync replication at GTID set %v error %v", gset, err) } - log.Infof("start sync binlog at GTID %v", gtid) + log.Infof("start sync binlog at GTID set %v", gset) return s, nil } } @@ -91,6 +91,9 @@ func (c *Canal) runSyncBinlog() error { } continue case *replication.XIDEvent: + if e.GSet != nil { + c.master.UpdateGTIDSet(e.GSet) + } savePos = true // try to save the position later if err := c.eventHandler.OnXID(pos); err != nil { @@ -102,8 +105,6 @@ func (c *Canal) runSyncBinlog() error { if err != nil { return errors.Trace(err) } - - c.master.UpdateGTID(gtid) if err := c.eventHandler.OnGTID(gtid); err != nil { return errors.Trace(err) } @@ -113,11 +114,13 @@ func (c *Canal) runSyncBinlog() error { if err != nil { return errors.Trace(err) } - c.master.UpdateGTID(gtid) if err := c.eventHandler.OnGTID(gtid); err != nil { return errors.Trace(err) } case *replication.QueryEvent: + if e.GSet != nil { + c.master.UpdateGTIDSet(e.GSet) + } var ( mb [][]byte schema []byte @@ -194,6 +197,11 @@ func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error { return c.eventHandler.OnRow(events) } +func (c *Canal) FlushBinlog() error { + _, err := c.Execute("FLUSH BINARY LOGS") + return errors.Trace(err) +} + func (c *Canal) WaitUntilPos(pos mysql.Position, timeout time.Duration) error { timer := time.NewTimer(timeout) for { @@ -201,6 +209,10 @@ func (c *Canal) WaitUntilPos(pos mysql.Position, timeout time.Duration) error { case <-timer.C: return errors.Errorf("wait position %v too long > %s", pos, timeout) default: + err := c.FlushBinlog() + if err != nil { + return errors.Trace(err) + } curPos := c.master.Position() if curPos.Compare(pos) >= 0 { return nil @@ -226,6 +238,29 @@ func (c *Canal) GetMasterPos() (mysql.Position, error) { return mysql.Position{name, uint32(pos)}, nil } +func (c *Canal) GetMasterGTIDSet() (mysql.GTIDSet, error) { + query := "" + switch c.cfg.Flavor { + case mysql.MariaDBFlavor: + query = "SELECT @@GLOBAL.gtid_current_pos" + default: + query = "SELECT @@GLOBAL.GTID_EXECUTED" + } + rr, err := c.Execute(query) + if err != nil { + return nil, errors.Trace(err) + } + gx, err := rr.GetString(0, 0) + if err != nil { + return nil, errors.Trace(err) + } + gset, err := mysql.ParseGTIDSet(c.cfg.Flavor, gx) + if err != nil { + return nil, errors.Trace(err) + } + return gset, nil +} + func (c *Canal) CatchMasterPos(timeout time.Duration) error { pos, err := c.GetMasterPos() if err != nil { diff --git a/vendor/github.com/siddontang/go-mysql/dump/dump.go b/vendor/github.com/siddontang/go-mysql/dump/dump.go index 29344406..829a3020 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/dump.go +++ b/vendor/github.com/siddontang/go-mysql/dump/dump.go @@ -147,9 +147,6 @@ func (d *Dumper) Dump(w io.Writer) error { // Multi row is easy for us to parse the data args = append(args, "--skip-extended-insert") - // Disable gtid purge - args = append(args, "--set-gtid-purged=OFF") - for db, tables := range d.IgnoreTables { for _, table := range tables { args = append(args, fmt.Sprintf("--ignore-table=%s.%s", db, table)) diff --git a/vendor/github.com/siddontang/go-mysql/mysql/resultset.go b/vendor/github.com/siddontang/go-mysql/mysql/resultset.go index a50d9f89..08040508 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/resultset.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/resultset.go @@ -115,7 +115,8 @@ func (p RowData) ParseBinary(f []*Field) ([]interface{}, error) { } else { data[i] = ParseBinaryInt24(p[pos : pos+3]) } - pos += 4 + //3 byte + pos += 3 continue case MYSQL_TYPE_LONG: diff --git a/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go b/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go index 1a43d43a..307684db 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go @@ -79,26 +79,40 @@ func formatBinaryValue(value interface{}) ([]byte, error) { return nil, errors.Errorf("invalid type %T", value) } } + +func fieldType(value interface{}) (typ uint8, err error) { + switch value.(type) { + case int8, int16, int32, int64, int: + typ = MYSQL_TYPE_LONGLONG + case uint8, uint16, uint32, uint64, uint: + typ = MYSQL_TYPE_LONGLONG + case float32, float64: + typ = MYSQL_TYPE_DOUBLE + case string, []byte: + typ = MYSQL_TYPE_VAR_STRING + case nil: + typ = MYSQL_TYPE_NULL + default: + err = errors.Errorf("unsupport type %T for resultset", value) + } + return +} + func formatField(field *Field, value interface{}) error { switch value.(type) { case int8, int16, int32, int64, int: field.Charset = 63 - field.Type = MYSQL_TYPE_LONGLONG field.Flag = BINARY_FLAG | NOT_NULL_FLAG case uint8, uint16, uint32, uint64, uint: field.Charset = 63 - field.Type = MYSQL_TYPE_LONGLONG field.Flag = BINARY_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG case float32, float64: field.Charset = 63 - field.Type = MYSQL_TYPE_DOUBLE field.Flag = BINARY_FLAG | NOT_NULL_FLAG case string, []byte: field.Charset = 33 - field.Type = MYSQL_TYPE_VAR_STRING case nil: field.Charset = 33 - field.Type = MYSQL_TYPE_NULL default: return errors.Errorf("unsupport type %T for resultset", value) } @@ -111,7 +125,13 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse r.Fields = make([]*Field, len(names)) var b []byte - var err error + + if len(values) == 0 { + for i, name := range names { + r.Fields[i] = &Field{Name: hack.Slice(name), Charset: 33, Type: MYSQL_TYPE_NULL} + } + return r, nil + } for i, vs := range values { if len(vs) != len(r.Fields) { @@ -120,13 +140,23 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse var row []byte for j, value := range vs { - if i == 0 { - field := &Field{} - r.Fields[j] = field - field.Name = hack.Slice(names[j]) - - if err = formatField(field, value); err != nil { - return nil, errors.Trace(err) + typ, err := fieldType(value) + if err != nil { + return nil, errors.Trace(err) + } + if r.Fields[j] == nil { + r.Fields[j] = &Field{Name: hack.Slice(names[j]), Type: typ} + formatField(r.Fields[j], value) + } else if typ != r.Fields[j].Type { + // we got another type in the same column. in general, we treat it as an error, except + // the case, when old value was null, and the new one isn't null, so we can update + // type info for fields. + oldIsNull, newIsNull := r.Fields[j].Type == MYSQL_TYPE_NULL, typ == MYSQL_TYPE_NULL + if oldIsNull && !newIsNull { // old is null, new isn't, update type info. + r.Fields[j].Type = typ + formatField(r.Fields[j], value) + } else if !oldIsNull && !newIsNull { // different non-null types, that's an error. + return nil, errors.Errorf("row types aren't consistent") } } b, err = formatTextValue(value) @@ -155,7 +185,6 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result r.Fields = make([]*Field, len(names)) var b []byte - var err error bitmapLen := ((len(names) + 7 + 2) >> 3) @@ -171,8 +200,12 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result row = append(row, nullBitmap...) for j, value := range vs { + typ, err := fieldType(value) + if err != nil { + return nil, errors.Trace(err) + } if i == 0 { - field := &Field{} + field := &Field{Type: typ} r.Fields[j] = field field.Name = hack.Slice(names[j]) diff --git a/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go b/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go index 84fe7f98..2dcd0c16 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go +++ b/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go @@ -4,7 +4,7 @@ import ( "context" "github.com/juju/errors" - "gopkg.in/birkirb/loggers.v1/log" + "github.com/siddontang/go-log/log" ) var ( diff --git a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go index c0ac2826..b4913c97 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go +++ b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go @@ -12,9 +12,9 @@ import ( "github.com/juju/errors" "github.com/satori/go.uuid" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/client" . "github.com/siddontang/go-mysql/mysql" - "gopkg.in/birkirb/loggers.v1/log" ) var ( @@ -101,6 +101,10 @@ type BinlogSyncer struct { // NewBinlogSyncer creates the BinlogSyncer with cfg. func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { + if cfg.ServerID == 0 { + log.Fatal("can't use 0 as the server ID") + } + // Clear the Password to avoid outputing it in log. pass := cfg.Password cfg.Password = "" @@ -334,7 +338,7 @@ func (b *BinlogSyncer) StartSync(pos Position) (*BinlogStreamer, error) { // StartSyncGTID starts syncing from the `gset` GTIDSet. func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { - log.Infof("begin to sync binlog from GTID %s", gset) + log.Infof("begin to sync binlog from GTID set %s", gset) b.gset = gset diff --git a/vendor/github.com/siddontang/go-mysql/replication/parser.go b/vendor/github.com/siddontang/go-mysql/replication/parser.go index dcfbe6ed..ad325921 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/parser.go +++ b/vendor/github.com/siddontang/go-mysql/replication/parser.go @@ -71,7 +71,9 @@ func (p *BinlogParser) ParseFile(name string, offset int64, onEvent OnEventFunc) return errors.Errorf("seek %s to %d error %v", name, offset, err) } - p.getFormatDescriptionEvent(f, onEvent) + if err = p.parseFormatDescriptionEvent(f, onEvent); err != nil { + return errors.Annotatef(err, "parse FormatDescriptionEvent") + } } if _, err = f.Seek(offset, os.SEEK_SET); err != nil { @@ -81,18 +83,23 @@ func (p *BinlogParser) ParseFile(name string, offset int64, onEvent OnEventFunc) return p.ParseReader(f, onEvent) } -func (p *BinlogParser) getFormatDescriptionEvent(r io.Reader, onEvent OnEventFunc) error { - _, err := p.parseSingleEvent(&r, onEvent) +func (p *BinlogParser) parseFormatDescriptionEvent(r io.Reader, onEvent OnEventFunc) error { + _, err := p.parseSingleEvent(r, onEvent) return err } -func (p *BinlogParser) parseSingleEvent(r *io.Reader, onEvent OnEventFunc) (bool, error) { +// ParseSingleEvent parses single binlog event and passes the event to onEvent function. +func (p *BinlogParser) ParseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool, error) { + return p.parseSingleEvent(r, onEvent) +} + +func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool, error) { var err error var n int64 headBuf := make([]byte, EventHeaderSize) - if _, err = io.ReadFull(*r, headBuf); err == io.EOF { + if _, err = io.ReadFull(r, headBuf); err == io.EOF { return true, nil } else if err != nil { return false, errors.Trace(err) @@ -109,7 +116,7 @@ func (p *BinlogParser) parseSingleEvent(r *io.Reader, onEvent OnEventFunc) (bool } var buf bytes.Buffer - if n, err = io.CopyN(&buf, *r, int64(h.EventSize)-int64(EventHeaderSize)); err != nil { + if n, err = io.CopyN(&buf, r, int64(h.EventSize)-int64(EventHeaderSize)); err != nil { return false, errors.Errorf("get event body err %v, need %d - %d, but got %d", err, h.EventSize, EventHeaderSize, n) } @@ -145,7 +152,7 @@ func (p *BinlogParser) ParseReader(r io.Reader, onEvent OnEventFunc) error { break } - done, err := p.parseSingleEvent(&r, onEvent) + done, err := p.parseSingleEvent(r, onEvent) if err != nil { if _, ok := err.(errMissingTableMapEvent); ok { continue diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index a213fa6f..6d4d972c 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -11,9 +11,9 @@ import ( "github.com/juju/errors" "github.com/shopspring/decimal" + "github.com/siddontang/go-log/log" . "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go/hack" - "gopkg.in/birkirb/loggers.v1/log" ) type errMissingTableMapEvent error @@ -419,23 +419,31 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_TIMESTAMP: n = 4 t := binary.LittleEndian.Uint32(data) - v = e.parseFracTime(fracTime{time.Unix(int64(t), 0), 0}) + if t == 0 { + v = formatZeroTime(0, 0) + } else { + v = e.parseFracTime(fracTime{time.Unix(int64(t), 0), 0}) + } case MYSQL_TYPE_TIMESTAMP2: v, n, err = decodeTimestamp2(data, meta) v = e.parseFracTime(v) case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) - d := i64 / 1000000 - t := i64 % 1000000 - v = e.parseFracTime(fracTime{time.Date(int(d/10000), - time.Month((d%10000)/100), - int(d%100), - int(t/10000), - int((t%10000)/100), - int(t%100), - 0, - time.UTC), 0}) + if i64 == 0 { + v = formatZeroTime(0, 0) + } else { + d := i64 / 1000000 + t := i64 % 1000000 + v = e.parseFracTime(fracTime{time.Date(int(d/10000), + time.Month((d%10000)/100), + int(d%100), + int(t/10000), + int((t%10000)/100), + int(t%100), + 0, + time.UTC), 0}) + } case MYSQL_TYPE_DATETIME2: v, n, err = decodeDatetime2(data, meta) v = e.parseFracTime(v) diff --git a/vendor/gopkg.in/birkirb/loggers.v1/log/log.go b/vendor/gopkg.in/birkirb/loggers.v1/log/log.go deleted file mode 100644 index 90d9f22d..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/log/log.go +++ /dev/null @@ -1,141 +0,0 @@ -package log - -import ( - "gopkg.in/birkirb/loggers.v1" - "gopkg.in/birkirb/loggers.v1/mappers" - "gopkg.in/birkirb/loggers.v1/mappers/stdlib" -) - -// Logger is an Contextual logger interface. -var Logger loggers.Contextual - -func init() { - Logger = stdlib.NewDefaultLogger() -} - -// ParseLevel parses the level -func ParseLevel(s string) mappers.Level { - return mappers.ParseLevel(s) -} - -// SetLevel changes the level -func SetLevel(l mappers.Level) { - if logger, ok := Logger.(mappers.LevelSetter); ok { - logger.SetLevel(l) - } -} - -// Debug should be used when logging exessive debug info. -func Debug(v ...interface{}) { - Logger.Debug(v...) -} - -// Debugf works the same as Debug but supports formatting. -func Debugf(format string, v ...interface{}) { - Logger.Debugf(format, v...) -} - -// Debugln works the same as Debug but supports formatting. -func Debugln(v ...interface{}) { - Logger.Debugln(v...) -} - -// Info is a general function to log something. -func Info(v ...interface{}) { - Logger.Info(v...) -} - -// Infof works the same as Info but supports formatting. -func Infof(format string, v ...interface{}) { - Logger.Infof(format, v...) -} - -// Infoln works the same as Info but supports formatting. -func Infoln(v ...interface{}) { - Logger.Infoln(v...) -} - -// Warn is useful for alerting about something wrong. -func Warn(v ...interface{}) { - Logger.Warn(v...) -} - -// Warnf works the same as Warn but supports formatting. -func Warnf(format string, v ...interface{}) { - Logger.Warnf(format, v...) -} - -// Warnln works the same as Warn but prints each value on a line. -func Warnln(v ...interface{}) { - Logger.Warnln(v...) -} - -// Error should be used only if real error occures. -func Error(v ...interface{}) { - Logger.Error(v...) -} - -// Errorf works the same as Error but supports formatting. -func Errorf(format string, v ...interface{}) { - Logger.Errorf(format, v...) -} - -// Errorln works the same as Error but prints each value on a line. -func Errorln(v ...interface{}) { - Logger.Errorln(v...) -} - -// Fatal should be only used when it's not possible to continue program execution. -func Fatal(v ...interface{}) { - Logger.Fatal(v...) -} - -// Fatalf works the same as Fatal but supports formatting. -func Fatalf(format string, v ...interface{}) { - Logger.Fatalf(format, v...) -} - -// Fatalln works the same as Fatal but prints each value on a line. -func Fatalln(v ...interface{}) { - Logger.Fatalln(v...) -} - -// Panic should be used only if real panic is desired. -func Panic(v ...interface{}) { - Logger.Panic(v...) -} - -// Panicf works the same as Panic but supports formatting. -func Panicf(format string, v ...interface{}) { - Logger.Panicf(format, v...) -} - -// Panicln works the same as Panic but prints each value on a line. -func Panicln(v ...interface{}) { - Logger.Panicln(v...) -} - -// Print should be used for information messages. -func Print(v ...interface{}) { - Logger.Print(v...) -} - -// Printf works the same as Print but supports formatting. -func Printf(format string, v ...interface{}) { - Logger.Printf(format, v...) -} - -// Println works the same as Print but prints each value on a line. -func Println(v ...interface{}) { - Logger.Println(v...) -} - -// WithField adds the key value as parameter to log. -func WithField(key string, value interface{}) loggers.Advanced { - return Logger.WithField(key, value) -} - -// WithFields adds the fields as a list of key/value parameters to log. Even number expected. -func WithFields(fields ...interface{}) loggers.Advanced { - return Logger.WithFields(fields...) -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/advanced.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/advanced.go deleted file mode 100644 index 97823264..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/advanced.go +++ /dev/null @@ -1,77 +0,0 @@ -package mappers - -// AdvancedMap maps a standard logger to an advanced logger interface. -type AdvancedMap struct { - standardMap -} - -// NewAdvancedMap returns an advanced logger that is mapped via mapper. -func NewAdvancedMap(m LevelMapper) *AdvancedMap { - var a AdvancedMap - - if m != nil { - a.LevelMapper = m - } - - return &a -} - -// Debug should be used when logging exessive debug info. -func (a *AdvancedMap) Debug(v ...interface{}) { - a.LevelPrint(LevelDebug, v...) -} - -// Debugf works the same as Debug but supports formatting. -func (a *AdvancedMap) Debugf(format string, v ...interface{}) { - a.LevelPrintf(LevelDebug, format, v...) -} - -// Debugln works the same as Debug but supports formatting. -func (a *AdvancedMap) Debugln(v ...interface{}) { - a.LevelPrintln(LevelDebug, v...) -} - -// Info is a general function to log something. -func (a *AdvancedMap) Info(v ...interface{}) { - a.LevelPrint(LevelInfo, v...) -} - -// Infof works the same as Info but supports formatting. -func (a *AdvancedMap) Infof(format string, v ...interface{}) { - a.LevelPrintf(LevelInfo, format, v...) -} - -// Infoln works the same as Info but supports formatting. -func (a *AdvancedMap) Infoln(v ...interface{}) { - a.LevelPrintln(LevelInfo, v...) -} - -// Warn is useful for alerting about something wrong. -func (a *AdvancedMap) Warn(v ...interface{}) { - a.LevelPrint(LevelWarn, v...) -} - -// Warnf works the same as Warn but supports formatting. -func (a *AdvancedMap) Warnf(format string, v ...interface{}) { - a.LevelPrintf(LevelWarn, format, v...) -} - -// Warnln works the same as Warn but supports formatting. -func (a *AdvancedMap) Warnln(v ...interface{}) { - a.LevelPrintln(LevelWarn, v...) -} - -// Error should be used only if real error occures. -func (a *AdvancedMap) Error(v ...interface{}) { - a.LevelPrint(LevelError, v...) -} - -// Errorf works the same as Error but supports formatting. -func (a *AdvancedMap) Errorf(format string, v ...interface{}) { - a.LevelPrintf(LevelError, format, v...) -} - -// Errorln works the same as Error but supports formatting. -func (a *AdvancedMap) Errorln(v ...interface{}) { - a.LevelPrintln(LevelError, v...) -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/contextual.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/contextual.go deleted file mode 100644 index 632f05c8..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/contextual.go +++ /dev/null @@ -1,33 +0,0 @@ -package mappers - -import "gopkg.in/birkirb/loggers.v1" - -// ContextualMap maps a logger to a contextual logger interface. -type ContextualMap struct { - AdvancedMap - ContextualMapper -} - -// NewContextualMap returns an contextual logger that is mapped via mapper. -func NewContextualMap(m ContextualMapper) *ContextualMap { - var a ContextualMap - - if m != nil { - if am := NewAdvancedMap(m); am != nil { - a.AdvancedMap = *am - } - a.ContextualMapper = m - } - - return &a -} - -// WithField directly maps the loggers method. -func (c *ContextualMap) WithField(key string, value interface{}) loggers.Advanced { - return c.ContextualMapper.WithField(key, value) -} - -// WithFields directly maps the loggers method. -func (c *ContextualMap) WithFields(fields ...interface{}) loggers.Advanced { - return c.ContextualMapper.WithFields(fields...) -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/mappers.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/mappers.go deleted file mode 100644 index ab9a5919..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/mappers.go +++ /dev/null @@ -1,85 +0,0 @@ -package mappers - -import ( - "strings" - - "gopkg.in/birkirb/loggers.v1" -) - -type ( - // Level indicates a specific log level. - Level byte - - // LevelMapper interfaces allows a logger to map to any Advanced Logger. - LevelMapper interface { - LevelPrint(Level, ...interface{}) - LevelPrintf(Level, string, ...interface{}) - LevelPrintln(Level, ...interface{}) - } - - // ContextualMapper interfaces allows a logger to map to any Contextual Logger. - ContextualMapper interface { - LevelMapper - WithField(key string, value interface{}) loggers.Advanced - WithFields(fields ...interface{}) loggers.Advanced - } - - // LevelSetter changes the level - LevelSetter interface { - SetLevel(l Level) - } -) - -const ( - // LevelDebug is a log Level. - LevelDebug Level = iota - // LevelInfo is a log Level. - LevelInfo - // LevelWarn is a log Level. - LevelWarn - // LevelError is a log Level. - LevelError - // LevelFatal is a log Level. - LevelFatal - // LevelPanic is a log Level. - LevelPanic -) - -func (l Level) String() string { - switch l { - case LevelDebug: - return "DEBUG " - case LevelInfo: - return "INFO " - case LevelWarn: - return "WARN " - case LevelError: - return "ERROR " - case LevelFatal: - return "FATAL " - case LevelPanic: - return "PANIC " - default: - panic("Missing case statement in Level String.") - } -} - -// ParseLevel parses the level from string. Return default Info Level if error. -func ParseLevel(s string) Level { - switch strings.ToLower(s) { - case "debug": - return LevelDebug - case "info": - return LevelInfo - case "warn", "warning": - return LevelWarn - case "error": - return LevelError - case "fatal": - return LevelFatal - case "panic": - return LevelPanic - default: - return LevelInfo - } -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/standard.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/standard.go deleted file mode 100644 index 797ce34f..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/standard.go +++ /dev/null @@ -1,63 +0,0 @@ -package mappers - -import ( - "errors" - "fmt" - "os" -) - -type standardMap struct { - LevelMapper -} - -// Print should be used only if real error occures. -func (s *standardMap) Print(v ...interface{}) { - s.LevelPrint(LevelInfo, v...) -} - -// Printf works the same as Print but supports formatting. -func (s *standardMap) Printf(format string, v ...interface{}) { - s.LevelPrintf(LevelInfo, format, v...) -} - -// Println works the same as Print but supports formatting. -func (s *standardMap) Println(v ...interface{}) { - s.LevelPrintln(LevelInfo, v...) -} - -// Fatal works the same as Error but it terminates the program right after logging. -// Fatal should be only used when it's not possible to continue program execution. -func (s *standardMap) Fatal(v ...interface{}) { - s.LevelPrint(LevelFatal, v...) - os.Exit(1) -} - -// Fatalf works the same as Fatal but supports formatting. -func (s *standardMap) Fatalf(format string, v ...interface{}) { - s.LevelPrintf(LevelFatal, format, v...) - os.Exit(1) -} - -// Fatalln works the same as Info but supports formatting. -func (s *standardMap) Fatalln(v ...interface{}) { - s.LevelPrintln(LevelFatal, v...) - os.Exit(1) -} - -// Panic works the same as Error but it terminates the program right after logging. -func (s *standardMap) Panic(v ...interface{}) { - s.LevelPrint(LevelPanic, v...) - panic(errors.New(fmt.Sprint(v...))) -} - -// Panicf works the same as Panic but supports formatting. -func (s *standardMap) Panicf(format string, v ...interface{}) { - s.LevelPrintf(LevelPanic, format, v...) - panic(fmt.Errorf(format, v...)) -} - -// Panicln works the same as Panic but supports formatting. -func (s *standardMap) Panicln(v ...interface{}) { - s.LevelPrintln(LevelPanic, v...) - panic(errors.New(fmt.Sprint(v...))) -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/stdlib.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/stdlib.go deleted file mode 100644 index 149d62bc..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/stdlib.go +++ /dev/null @@ -1,123 +0,0 @@ -package stdlib - -import ( - "fmt" - "log" - "os" - "strings" - - "gopkg.in/birkirb/loggers.v1" - "gopkg.in/birkirb/loggers.v1/mappers" -) - -// goLog maps the standard log package logger to an Advanced log interface. -// However it mostly ignores any level info. -type goLog struct { - logger *log.Logger - level mappers.Level -} - -// NewDefaultLogger returns a Contextual logger using a log.Logger with stderr output. -func NewDefaultLogger() loggers.Contextual { - var g goLog - g.logger = log.New(os.Stderr, "", log.Ldate|log.Ltime) - g.level = mappers.LevelInfo - - a := mappers.NewContextualMap(&g) - a.Debug("Now using Go's stdlib log package (via loggers/mappers/stdlib).") - - return a -} - -// NewLogger creates a Contextual logger from a log.Logger. -func NewLogger(l *log.Logger) loggers.Contextual { - var g goLog - g.logger = l - - a := mappers.NewContextualMap(&g) - a.Debug("Now using Go's stdlib log package (via loggers/mappers/stdlib).") - - return a -} - -// SetLevel implements LevelSetter interface -func (l *goLog) SetLevel(level mappers.Level) { - l.level = level -} - -// LevelPrint is a Mapper method -func (l *goLog) LevelPrint(lev mappers.Level, i ...interface{}) { - if l.level > lev { - return - } - - v := []interface{}{lev} - v = append(v, i...) - l.logger.Print(v...) -} - -// LevelPrintf is a Mapper method -func (l *goLog) LevelPrintf(lev mappers.Level, format string, i ...interface{}) { - if l.level > lev { - return - } - - f := "%s" + format - v := []interface{}{lev} - v = append(v, i...) - l.logger.Printf(f, v...) -} - -// LevelPrintln is a Mapper method -func (l *goLog) LevelPrintln(lev mappers.Level, i ...interface{}) { - if l.level > lev { - return - } - - v := []interface{}{lev} - v = append(v, i...) - l.logger.Println(v...) -} - -// WithField returns an advanced logger with a pre-set field. -func (l *goLog) WithField(key string, value interface{}) loggers.Advanced { - return l.WithFields(key, value) -} - -// WithFields returns an advanced logger with pre-set fields. -func (l *goLog) WithFields(fields ...interface{}) loggers.Advanced { - s := make([]string, 0, len(fields)/2) - for i := 0; i+1 < len(fields); i = i + 2 { - key := fields[i] - value := fields[i+1] - s = append(s, fmt.Sprint(key, "=", value)) - } - - r := gologPostfixLogger{l, "[" + strings.Join(s, ", ") + "]"} - return mappers.NewAdvancedMap(&r) -} - -type gologPostfixLogger struct { - *goLog - postfix string -} - -func (r *gologPostfixLogger) LevelPrint(lev mappers.Level, i ...interface{}) { - if len(r.postfix) > 0 { - i = append(i, " ", r.postfix) - } - r.goLog.LevelPrint(lev, i...) -} - -func (r *gologPostfixLogger) LevelPrintf(lev mappers.Level, format string, i ...interface{}) { - if len(r.postfix) > 0 { - format = format + " %s" - i = append(i, r.postfix) - } - r.goLog.LevelPrintf(lev, format, i...) -} - -func (r *gologPostfixLogger) LevelPrintln(lev mappers.Level, i ...interface{}) { - i = append(i, r.postfix) - r.goLog.LevelPrintln(lev, i...) -} diff --git a/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/testing.go b/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/testing.go deleted file mode 100644 index 5bb02644..00000000 --- a/vendor/gopkg.in/birkirb/loggers.v1/mappers/stdlib/testing.go +++ /dev/null @@ -1,93 +0,0 @@ -package stdlib - -import ( - "fmt" - "strings" - "testing" - - "gopkg.in/birkirb/loggers.v1" - "gopkg.in/birkirb/loggers.v1/mappers" -) - -// goTestLog maps the testing logger to an Advanced log interface. -// However it ignores any level info. -type goTestLog struct { - logger *testing.T -} - -// NewDefaultLogger returns a Contextual logger using a *testing.T with Log/Logf output. -// This allows logging to be redirected to the test where it belongs. -func NewTestingLogger(t *testing.T) loggers.Contextual { - var g goTestLog - g.logger = t - - a := mappers.NewContextualMap(&g) - a.Debug("Now using Go's stdlib testing log (via loggers/mappers/stdlib).") - - return a -} - -// LevelPrint is a Mapper method -func (l *goTestLog) LevelPrint(lev mappers.Level, i ...interface{}) { - v := []interface{}{lev} - v = append(v, i...) - l.logger.Log(v...) -} - -// LevelPrintf is a Mapper method -func (l *goTestLog) LevelPrintf(lev mappers.Level, format string, i ...interface{}) { - f := "%s" + format - v := []interface{}{lev} - v = append(v, i...) - l.logger.Logf(f, v...) -} - -// LevelPrintln is a Mapper method -func (l *goTestLog) LevelPrintln(lev mappers.Level, i ...interface{}) { - v := []interface{}{lev} - v = append(v, i...) - l.logger.Log(v...) -} - -// WithField returns an advanced logger with a pre-set field. -func (l *goTestLog) WithField(key string, value interface{}) loggers.Advanced { - return l.WithFields(key, value) -} - -// WithFields returns an advanced logger with pre-set fields. -func (l *goTestLog) WithFields(fields ...interface{}) loggers.Advanced { - s := make([]string, 0, len(fields)/2) - for i := 0; i+1 < len(fields); i = i + 2 { - key := fields[i] - value := fields[i+1] - s = append(s, fmt.Sprint(key, "=", value)) - } - - r := goTestLogPostfixLogger{l, "["+strings.Join(s, ", ")+"]"} - return mappers.NewAdvancedMap(&r) -} - -type goTestLogPostfixLogger struct { - *goTestLog - postfix string -} - -func (r *goTestLogPostfixLogger) LevelPrint(lev mappers.Level, i ...interface{}) { - if len(r.postfix) > 0 { - i = append(i, " ", r.postfix) - } - r.goTestLog.LevelPrint(lev, i...) -} - -func (r *goTestLogPostfixLogger) LevelPrintf(lev mappers.Level, format string, i ...interface{}) { - if len(r.postfix) > 0 { - format = format + " %s" - i = append(i, r.postfix) - } - r.goTestLog.LevelPrintf(lev, format, i...) -} - -func (r *goTestLogPostfixLogger) LevelPrintln(lev mappers.Level, i ...interface{}) { - i = append(i, r.postfix) - r.goTestLog.LevelPrintln(lev, i...) -} From 5cf0adf53b4750cdfe360ad9883eedeb3a01d31f Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 28 Jul 2018 22:12:51 +0800 Subject: [PATCH 2/4] fix compile --- cmd/go-mysql-elasticsearch/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/go-mysql-elasticsearch/main.go b/cmd/go-mysql-elasticsearch/main.go index 9c8856a9..af035075 100644 --- a/cmd/go-mysql-elasticsearch/main.go +++ b/cmd/go-mysql-elasticsearch/main.go @@ -27,8 +27,7 @@ func main() { runtime.GOMAXPROCS(runtime.NumCPU()) flag.Parse() - level := log.ParseLevel(*logLevel) - log.SetLevel(level) + log.SetLevelByName(*logLevel) sc := make(chan os.Signal, 1) signal.Notify(sc, From d91fb4530945ee45a1e8164739e067db25121964 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 30 Jul 2018 09:29:06 +0800 Subject: [PATCH 3/4] update vendor to try to fix CI --- vendor/github.com/siddontang/go-mysql/dump/parser.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vendor/github.com/siddontang/go-mysql/dump/parser.go b/vendor/github.com/siddontang/go-mysql/dump/parser.go index ad409255..3e23600d 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/parser.go +++ b/vendor/github.com/siddontang/go-mysql/dump/parser.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/juju/errors" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" ) @@ -79,11 +80,13 @@ func Parse(r io.Reader, h ParseHandler, parseBinlogPos bool) error { values, err := parseValues(m[0][2]) if err != nil { - return errors.Errorf("parse values %v err", line) + return errors.Errorf("parse values %v err %v", line, err) } if err = h.Data(db, table, values); err != nil && err != ErrSkip { return errors.Trace(err) + } else if err == ErrSkip { + log.Errorf("handle data %v err", line) } } } From 921eb411d9b111214eed987e8d441e3a0e061f74 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 30 Jul 2018 15:02:43 +0800 Subject: [PATCH 4/4] update go-mysql vendor --- Gopkg.lock | 2 +- .../siddontang/go-mysql/canal/canal.go | 2 ++ .../siddontang/go-mysql/canal/dump.go | 9 ++++++ .../siddontang/go-mysql/dump/dump.go | 28 +++++++++++++------ .../siddontang/go-mysql/dump/parser.go | 5 +--- .../siddontang/go-mysql/replication/backup.go | 2 +- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d8827cc9..38021a8a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -62,7 +62,7 @@ "replication", "schema" ] - revision = "51eceecc377e5de21b7e5f43d3fb4218d305ac4a" + revision = "2d151e326c1a7193d6c374dba6fbb0db3435bf05" [solve-meta] analyzer-name = "dep" diff --git a/vendor/github.com/siddontang/go-mysql/canal/canal.go b/vendor/github.com/siddontang/go-mysql/canal/canal.go index 2f86adc6..1303fd5a 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/canal.go +++ b/vendor/github.com/siddontang/go-mysql/canal/canal.go @@ -147,6 +147,8 @@ func (c *Canal) prepareDumper() error { c.dumper.SetWhere(c.cfg.Dump.Where) c.dumper.SkipMasterData(c.cfg.Dump.SkipMasterData) c.dumper.SetMaxAllowedPacket(c.cfg.Dump.MaxAllowedPacketMB) + // Use hex blob for mysqldump + c.dumper.SetHexBlob(true) for _, ignoreTable := range c.cfg.Dump.IgnoreTables { if seps := strings.Split(ignoreTable, ","); len(seps) == 2 { diff --git a/vendor/github.com/siddontang/go-mysql/canal/dump.go b/vendor/github.com/siddontang/go-mysql/canal/dump.go index 212cdaac..9cf153d7 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/dump.go +++ b/vendor/github.com/siddontang/go-mysql/canal/dump.go @@ -1,8 +1,10 @@ package canal import ( + "encoding/hex" "fmt" "strconv" + "strings" "time" "github.com/juju/errors" @@ -62,6 +64,13 @@ func (h *dumpParseHandler) Data(db string, table string, values []string) error return dump.ErrSkip } vs[i] = f + } else if strings.HasPrefix(v, "0x") { + buf, err := hex.DecodeString(v[2:]) + if err != nil { + log.Errorf("parse row %v at %d error %v, skip", values, i, err) + return dump.ErrSkip + } + vs[i] = string(buf) } else { log.Errorf("parse row %v error, invalid type at %d, skip", values, i) return dump.ErrSkip diff --git a/vendor/github.com/siddontang/go-mysql/dump/dump.go b/vendor/github.com/siddontang/go-mysql/dump/dump.go index 829a3020..8ebc4d65 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/dump.go +++ b/vendor/github.com/siddontang/go-mysql/dump/dump.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/juju/errors" + "github.com/siddontang/go-log/log" . "github.com/siddontang/go-mysql/mysql" ) @@ -35,6 +36,7 @@ type Dumper struct { masterDataSkipped bool maxAllowedPacket int + hexBlob bool } func NewDumper(executionPath string, addr string, user string, password string) (*Dumper, error) { @@ -84,6 +86,10 @@ func (d *Dumper) SetMaxAllowedPacket(i int) { d.maxAllowedPacket = i } +func (d *Dumper) SetHexBlob(v bool) { + d.hexBlob = v +} + func (d *Dumper) AddDatabases(dbs ...string) { d.Databases = append(d.Databases, dbs...) } @@ -147,12 +153,25 @@ func (d *Dumper) Dump(w io.Writer) error { // Multi row is easy for us to parse the data args = append(args, "--skip-extended-insert") + if d.hexBlob { + // Use hex for the binary type + args = append(args, "--hex-blob") + } + for db, tables := range d.IgnoreTables { for _, table := range tables { args = append(args, fmt.Sprintf("--ignore-table=%s.%s", db, table)) } } + if len(d.Charset) != 0 { + args = append(args, fmt.Sprintf("--default-character-set=%s", d.Charset)) + } + + if len(d.Where) != 0 { + args = append(args, fmt.Sprintf("--where=%s", d.Where)) + } + if len(d.Tables) == 0 && len(d.Databases) == 0 { args = append(args, "--all-databases") } else if len(d.Tables) == 0 { @@ -168,14 +187,7 @@ func (d *Dumper) Dump(w io.Writer) error { w.Write([]byte(fmt.Sprintf("USE `%s`;\n", d.TableDB))) } - if len(d.Charset) != 0 { - args = append(args, fmt.Sprintf("--default-character-set=%s", d.Charset)) - } - - if len(d.Where) != 0 { - args = append(args, fmt.Sprintf("--where=%s", d.Where)) - } - + log.Infof("exec mysqldump with %v", args) cmd := exec.Command(d.ExecutionPath, args...) cmd.Stderr = d.ErrOut diff --git a/vendor/github.com/siddontang/go-mysql/dump/parser.go b/vendor/github.com/siddontang/go-mysql/dump/parser.go index 3e23600d..ad409255 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/parser.go +++ b/vendor/github.com/siddontang/go-mysql/dump/parser.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/juju/errors" - "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" ) @@ -80,13 +79,11 @@ func Parse(r io.Reader, h ParseHandler, parseBinlogPos bool) error { values, err := parseValues(m[0][2]) if err != nil { - return errors.Errorf("parse values %v err %v", line, err) + return errors.Errorf("parse values %v err", line) } if err = h.Data(db, table, values); err != nil && err != ErrSkip { return errors.Trace(err) - } else if err == ErrSkip { - log.Errorf("handle data %v err", line) } } } diff --git a/vendor/github.com/siddontang/go-mysql/replication/backup.go b/vendor/github.com/siddontang/go-mysql/replication/backup.go index 01b2aef0..24a25aea 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/backup.go +++ b/vendor/github.com/siddontang/go-mysql/replication/backup.go @@ -40,7 +40,7 @@ func (b *BinlogSyncer) StartBackup(backupDir string, p Position, timeout time.Du }() for { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), timeout) e, err := s.GetEvent(ctx) cancel()