From f715f173558c13ba445bda7ad527f3cdb6b1351b Mon Sep 17 00:00:00 2001
From: huangjunwei <huangjunwei@youmi.net>
Date: Mon, 26 Feb 2024 00:20:54 +0800
Subject: [PATCH 1/4] tune slice append performance
---
canal/sync.go | 12 ++++++------
replication/binlogstreamer.go | 4 ++--
replication/event.go | 8 ++++----
replication/parser.go | 3 +--
replication/row_event.go | 24 +++++++++---------------
5 files changed, 22 insertions(+), 29 deletions(-)
diff --git a/canal/sync.go b/canal/sync.go
index b7006b17e..49cb1b59f 100644
--- a/canal/sync.go
+++ b/canal/sync.go
@@ -183,12 +183,12 @@ type node struct {
func parseStmt(stmt ast.StmtNode) (ns []*node) {
switch t := stmt.(type) {
case *ast.RenameTableStmt:
- for _, tableInfo := range t.TableToTables {
- n := &node{
+ ns = make([]*node, len(t.TableToTables))
+ for i, tableInfo := range t.TableToTables {
+ ns[i] = &node{
db: tableInfo.OldTable.Schema.String(),
table: tableInfo.OldTable.Name.String(),
}
- ns = append(ns, n)
}
case *ast.AlterTableStmt:
n := &node{
@@ -197,12 +197,12 @@ func parseStmt(stmt ast.StmtNode) (ns []*node) {
}
ns = []*node{n}
case *ast.DropTableStmt:
- for _, table := range t.Tables {
- n := &node{
+ ns = make([]*node, len(t.Tables))
+ for i, table := range t.Tables {
+ ns[i] = &node{
db: table.Schema.String(),
table: table.Name.String(),
}
- ns = append(ns, n)
}
case *ast.CreateTableStmt:
n := &node{
diff --git a/replication/binlogstreamer.go b/replication/binlogstreamer.go
index 72bc7ddd0..276e26722 100644
--- a/replication/binlogstreamer.go
+++ b/replication/binlogstreamer.go
@@ -60,9 +60,9 @@ func (s *BinlogStreamer) GetEventWithStartTime(ctx context.Context, startTime ti
// DumpEvents dumps all left events
func (s *BinlogStreamer) DumpEvents() []*BinlogEvent {
count := len(s.ch)
- events := make([]*BinlogEvent, 0, count)
+ events := make([]*BinlogEvent, count)
for i := 0; i < count; i++ {
- events = append(events, <-s.ch)
+ events[i] = <-s.ch
}
return events
}
diff --git a/replication/event.go b/replication/event.go
index 566efedaf..07896bec1 100644
--- a/replication/event.go
+++ b/replication/event.go
@@ -225,17 +225,17 @@ type PreviousGTIDsEvent struct {
}
func (e *PreviousGTIDsEvent) Decode(data []byte) error {
- var previousGTIDSets []string
pos := 0
uuidCount := binary.LittleEndian.Uint16(data[pos : pos+8])
pos += 8
+ previousGTIDSets := make([]string, uuidCount)
for i := uint16(0); i < uuidCount; i++ {
uuid := e.decodeUuid(data[pos : pos+16])
pos += 16
sliceCount := binary.LittleEndian.Uint16(data[pos : pos+8])
pos += 8
- var intervals []string
+ intervals := make([]string, sliceCount)
for i := uint16(0); i < sliceCount; i++ {
start := e.decodeInterval(data[pos : pos+8])
pos += 8
@@ -247,9 +247,9 @@ func (e *PreviousGTIDsEvent) Decode(data []byte) error {
} else {
interval = fmt.Sprintf("%d-%d", start, stop-1)
}
- intervals = append(intervals, interval)
+ intervals[i] = interval
}
- previousGTIDSets = append(previousGTIDSets, fmt.Sprintf("%s:%s", uuid, strings.Join(intervals, ":")))
+ previousGTIDSets[i] = fmt.Sprintf("%s:%s", uuid, strings.Join(intervals, ":"))
}
e.GTIDSets = strings.Join(previousGTIDSets, ",")
return nil
diff --git a/replication/parser.go b/replication/parser.go
index 4caf496c2..514eb8764 100644
--- a/replication/parser.go
+++ b/replication/parser.go
@@ -142,8 +142,7 @@ func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool,
return false, errors.Errorf("invalid raw data size in event %s, need %d but got %d", h.EventType, h.EventSize, buf.Len())
}
- var rawData []byte
- rawData = append(rawData, buf.Bytes()...)
+ rawData := buf.Bytes()
bodyLen := int(h.EventSize) - EventHeaderSize
body := rawData[EventHeaderSize:]
if len(body) != bodyLen {
diff --git a/replication/row_event.go b/replication/row_event.go
index 5828f8b77..a0d42e8ec 100644
--- a/replication/row_event.go
+++ b/replication/row_event.go
@@ -570,12 +570,9 @@ func (e *TableMapEvent) SetStrValueString() [][]string {
if len(e.SetStrValue) == 0 {
return nil
}
- e.setStrValueString = make([][]string, 0, len(e.SetStrValue))
- for _, vals := range e.SetStrValue {
- e.setStrValueString = append(
- e.setStrValueString,
- e.bytesSlice2StrSlice(vals),
- )
+ e.setStrValueString = make([][]string, len(e.SetStrValue))
+ for i, vals := range e.SetStrValue {
+ e.setStrValueString[i] = e.bytesSlice2StrSlice(vals)
}
}
return e.setStrValueString
@@ -588,12 +585,9 @@ func (e *TableMapEvent) EnumStrValueString() [][]string {
if len(e.EnumStrValue) == 0 {
return nil
}
- e.enumStrValueString = make([][]string, 0, len(e.EnumStrValue))
- for _, vals := range e.EnumStrValue {
- e.enumStrValueString = append(
- e.enumStrValueString,
- e.bytesSlice2StrSlice(vals),
- )
+ e.enumStrValueString = make([][]string, len(e.EnumStrValue))
+ for i, vals := range e.EnumStrValue {
+ e.enumStrValueString[i] = e.bytesSlice2StrSlice(vals)
}
}
return e.enumStrValueString
@@ -612,9 +606,9 @@ func (e *TableMapEvent) bytesSlice2StrSlice(src [][]byte) []string {
if src == nil {
return nil
}
- ret := make([]string, 0, len(src))
- for _, item := range src {
- ret = append(ret, string(item))
+ ret := make([]string, len(src))
+ for i, item := range src {
+ ret[i] = string(item)
}
return ret
}
From 7e2158e0b5b22958d3fa2717d979f51fb392d07a Mon Sep 17 00:00:00 2001
From: huangjunwei <huangjunwei@youmi.net>
Date: Fri, 1 Mar 2024 23:36:36 +0800
Subject: [PATCH 2/4] revert
---
replication/parser.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/replication/parser.go b/replication/parser.go
index 514eb8764..4caf496c2 100644
--- a/replication/parser.go
+++ b/replication/parser.go
@@ -142,7 +142,8 @@ func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool,
return false, errors.Errorf("invalid raw data size in event %s, need %d but got %d", h.EventType, h.EventSize, buf.Len())
}
- rawData := buf.Bytes()
+ var rawData []byte
+ rawData = append(rawData, buf.Bytes()...)
bodyLen := int(h.EventSize) - EventHeaderSize
body := rawData[EventHeaderSize:]
if len(body) != bodyLen {
From 6b06040b4d606c2f3b38d94eeecefa0450729003 Mon Sep 17 00:00:00 2001
From: huangjunwei <huangjunwei@youmi.net>
Date: Sat, 2 Mar 2024 00:03:19 +0800
Subject: [PATCH 3/4] tune raw data slice assignment
---
replication/parser.go | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/replication/parser.go b/replication/parser.go
index 4caf496c2..7e3a9370f 100644
--- a/replication/parser.go
+++ b/replication/parser.go
@@ -142,8 +142,10 @@ func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool,
return false, errors.Errorf("invalid raw data size in event %s, need %d but got %d", h.EventType, h.EventSize, buf.Len())
}
- var rawData []byte
- rawData = append(rawData, buf.Bytes()...)
+ rawData := make([]byte, buf.Len())
+ for i, b := range buf.Bytes() {
+ rawData[i] = b
+ }
bodyLen := int(h.EventSize) - EventHeaderSize
body := rawData[EventHeaderSize:]
if len(body) != bodyLen {
From 848265ed3ce18d13f77fc1208d29c1aa79f41c83 Mon Sep 17 00:00:00 2001
From: huangjunwei <huangjunwei@youmi.net>
Date: Mon, 4 Mar 2024 21:42:58 +0800
Subject: [PATCH 4/4] commit as suggested
---
replication/binlogstreamer.go | 2 +-
replication/event.go | 4 ++--
replication/parser.go | 6 ++----
3 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/replication/binlogstreamer.go b/replication/binlogstreamer.go
index 276e26722..5214e92dd 100644
--- a/replication/binlogstreamer.go
+++ b/replication/binlogstreamer.go
@@ -61,7 +61,7 @@ func (s *BinlogStreamer) GetEventWithStartTime(ctx context.Context, startTime ti
func (s *BinlogStreamer) DumpEvents() []*BinlogEvent {
count := len(s.ch)
events := make([]*BinlogEvent, count)
- for i := 0; i < count; i++ {
+ for i := range events {
events[i] = <-s.ch
}
return events
diff --git a/replication/event.go b/replication/event.go
index 07896bec1..b9e13bf1a 100644
--- a/replication/event.go
+++ b/replication/event.go
@@ -230,13 +230,13 @@ func (e *PreviousGTIDsEvent) Decode(data []byte) error {
pos += 8
previousGTIDSets := make([]string, uuidCount)
- for i := uint16(0); i < uuidCount; i++ {
+ for i := range previousGTIDSets {
uuid := e.decodeUuid(data[pos : pos+16])
pos += 16
sliceCount := binary.LittleEndian.Uint16(data[pos : pos+8])
pos += 8
intervals := make([]string, sliceCount)
- for i := uint16(0); i < sliceCount; i++ {
+ for i := range intervals {
start := e.decodeInterval(data[pos : pos+8])
pos += 8
stop := e.decodeInterval(data[pos : pos+8])
diff --git a/replication/parser.go b/replication/parser.go
index 7e3a9370f..4caf496c2 100644
--- a/replication/parser.go
+++ b/replication/parser.go
@@ -142,10 +142,8 @@ func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool,
return false, errors.Errorf("invalid raw data size in event %s, need %d but got %d", h.EventType, h.EventSize, buf.Len())
}
- rawData := make([]byte, buf.Len())
- for i, b := range buf.Bytes() {
- rawData[i] = b
- }
+ var rawData []byte
+ rawData = append(rawData, buf.Bytes()...)
bodyLen := int(h.EventSize) - EventHeaderSize
body := rawData[EventHeaderSize:]
if len(body) != bodyLen {