From ccabd266aaa188de9046ee782c144fec36fc7dab Mon Sep 17 00:00:00 2001 From: dongwook-chan Date: Tue, 15 Aug 2023 16:44:59 +0900 Subject: [PATCH 1/4] Add `VisibilityBitmap` to `TableMapEvent` for MySQL 8.0.23+ Invisible Columns Changes: - `TableMapEvent.VisibilityBitmap` `VisibilityBitmap` is a bitmap where each bit represents the visibility of a corresponding column in a table. If a bit is set, it indicates that the corresponding column is NOT an invinsible column. Invisible column was introduced in MySQL version 8.0.23. - `TableMapEvent.VisibilityMap` `VisibilityMap` lists boolean values of which is true if column of the same index is NOT an invisible column. Co-authored-by: sean --- replication/const.go | 1 + replication/row_event.go | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/replication/const.go b/replication/const.go index 75650e3f8..9c98d9323 100644 --- a/replication/const.go +++ b/replication/const.go @@ -240,6 +240,7 @@ const ( TABLE_MAP_OPT_META_PRIMARY_KEY_WITH_PREFIX TABLE_MAP_OPT_META_ENUM_AND_SET_DEFAULT_CHARSET TABLE_MAP_OPT_META_ENUM_AND_SET_COLUMN_CHARSET + TABLE_MAP_OPT_META_COLUMN_VISIBILITY ) type IntVarEventType byte diff --git a/replication/row_event.go b/replication/row_event.go index a7fad168b..42e1eb50b 100644 --- a/replication/row_event.go +++ b/replication/row_event.go @@ -81,6 +81,9 @@ type TableMapEvent struct { // EnumSetDefaultCharset/EnumSetColumnCharset is similar to DefaultCharset/ColumnCharset but for enum/set columns. EnumSetDefaultCharset []uint64 EnumSetColumnCharset []uint64 + + // VisibilityBitmap stores bits that are set if corresponding column is not invisible (MySQL 8.0.23+) + VisibilityBitmap []byte } func (e *TableMapEvent) Decode(data []byte) error { @@ -312,6 +315,9 @@ func (e *TableMapEvent) decodeOptionalMeta(data []byte) (err error) { return err } + case TABLE_MAP_OPT_META_COLUMN_VISIBILITY: + e.VisibilityBitmap = v + default: // Ignore for future extension } @@ -421,6 +427,7 @@ func (e *TableMapEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Primary key prefix: %v\n", e.PrimaryKeyPrefix) fmt.Fprintf(w, "Enum/set default charset: %v\n", e.EnumSetDefaultCharset) fmt.Fprintf(w, "Enum/set column charset: %v\n", e.EnumSetColumnCharset) + fmt.Fprintf(w, "Invisible Column bitmap: \n%s", hex.Dump(e.VisibilityBitmap)) unsignedMap := e.UnsignedMap() fmt.Fprintf(w, "UnsignedMap: %#v\n", unsignedMap) @@ -440,6 +447,9 @@ func (e *TableMapEvent) Dump(w io.Writer) { geometryTypeMap := e.GeometryTypeMap() fmt.Fprintf(w, "GeometryTypeMap: %#v\n", geometryTypeMap) + visibilityMap := e.VisibilityMap() + fmt.Fprintf(w, "VisibilityMap: %#v\n", visibilityMap) + nameMaxLen := 0 for _, name := range e.ColumnName { if len(name) > nameMaxLen { @@ -730,6 +740,22 @@ func (e *TableMapEvent) GeometryTypeMap() map[int]uint64 { return ret } +// VisibilityMap returns a map: column index -> visiblity. +// Invisible column was introduced in MySQL 8.0.23 +// nil is returned if not available. +func (e *TableMapEvent) VisibilityMap() map[int]bool { + if len(e.VisibilityBitmap) == 0 { + return nil + } + p := 0 + ret := make(map[int]bool) + for i := 0; i < int(e.ColumnCount); i++ { + ret[i] = e.VisibilityBitmap[p/8]&(1< Date: Wed, 16 Aug 2023 00:18:18 +0900 Subject: [PATCH 2/4] Refactor bitmap iteration to align with MySQL source code Changes: - Refactpred `UnsignedMap` - Refactored `VisibilityMap` Suggested by: https://github.com/go-mysql-org/go-mysql/pull/813#discussion_r1294333052 --- replication/row_event.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/replication/row_event.go b/replication/row_event.go index 42e1eb50b..df3f0380a 100644 --- a/replication/row_event.go +++ b/replication/row_event.go @@ -618,14 +618,19 @@ func (e *TableMapEvent) UnsignedMap() map[int]bool { if len(e.SignednessBitmap) == 0 { return nil } - p := 0 ret := make(map[int]bool) - for i := 0; i < int(e.ColumnCount); i++ { - if !e.IsNumericColumn(i) { - continue + i := 0 + for _, field := range e.SignednessBitmap { + for c := 0x80; c != 0; { + if e.IsNumericColumn(i) { + ret[i] = field&byte(c) != 0 + c >>= 1 + } + i++ + if i >= int(e.ColumnCount) { + return ret + } } - ret[i] = e.SignednessBitmap[p/8]&(1<>= 1 { + ret[i] = field&byte(c) != 0 + i++ + if uint64(i) >= e.ColumnCount { + return ret + } + } } return ret } From cac73e6efb9aa22d9c2ef98a4b81e7bcf7b8cbe6 Mon Sep 17 00:00:00 2001 From: dongwook-chan Date: Sun, 20 Aug 2023 02:48:21 +0900 Subject: [PATCH 3/4] Add test for column visibility in table map Changes: - Added data for MySQL 8.0 only (Only MySQL 8.0.23+ supports invisible columns) --- replication/row_event_test.go | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/replication/row_event_test.go b/replication/row_event_test.go index 8d71bf017..fe91ec4dd 100644 --- a/replication/row_event_test.go +++ b/replication/row_event_test.go @@ -939,6 +939,61 @@ func TestTableMapOptMetaPrimaryKey(t *testing.T) { } } +func TestTableMapOptMetaVisibility(t *testing.T) { + /* + CREATE TABLE _visibility( + `col0` INT INVISIBLE, + `col1` INT, + `col2` INT INVISIBLE, + `col3` INT, + `col4` INT, + `col5` INT INVISIBLE, + `col6` INT INVISIBLE, + `col7` INT INVISIBLE, + `col8` INT, + `col9` INT INVISIBLE, + `col10` INT INVISIBLE + ); + */ + case1VisibilityBitmap := []byte("X\x80") + case1VisibilityMap := map[int]bool{ + 0: false, + 1: true, + 2: false, + 3: true, + 4: true, + 5: false, + 6: false, + 7: false, + 8: true, + 9: false, + 10: false, + } + + // Invisible column and INVISIBLE keyword is available only on MySQL 8.0.23+ + testcases := []struct { + data []byte + expectedVisibilityBitmap []byte + expectedVisibilityMap map[int]bool + }{ + { + // mysql 8.0, case1 + data: []byte("^\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10\x0c\x02X\x80"), + expectedVisibilityBitmap: case1VisibilityBitmap, + expectedVisibilityMap: case1VisibilityMap, + }, + } + + for _, tc := range testcases { + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tc.data) + require.NoError(t, err) + require.Equal(t, tc.expectedVisibilityBitmap, tableMapEvent.VisibilityBitmap) + require.Equal(t, tc.expectedVisibilityMap, tableMapEvent.VisibilityMap()) + } +} + func TestTableMapHelperMaps(t *testing.T) { /* CREATE TABLE `_types` ( From b55ff1d81355684a7ad8f1fe7c1ea079650c494d Mon Sep 17 00:00:00 2001 From: will-k Date: Sun, 20 Aug 2023 16:56:07 +0900 Subject: [PATCH 4/4] Add test case 2 for column visibility in table map Changes: - Add test case 2 where invisible columns does not exists at all --- replication/row_event_test.go | 64 ++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/replication/row_event_test.go b/replication/row_event_test.go index fe91ec4dd..57ebdf623 100644 --- a/replication/row_event_test.go +++ b/replication/row_event_test.go @@ -940,6 +940,14 @@ func TestTableMapOptMetaPrimaryKey(t *testing.T) { } func TestTableMapOptMetaVisibility(t *testing.T) { + /* + SET GLOBAL binlog_row_image = FULL; + SET GLOBAL binlog_row_metadata = FULL; -- if applicable + + CREATE DATABASE test; + USE test; + */ + /* CREATE TABLE _visibility( `col0` INT INVISIBLE, @@ -955,7 +963,7 @@ func TestTableMapOptMetaVisibility(t *testing.T) { `col10` INT INVISIBLE ); */ - case1VisibilityBitmap := []byte("X\x80") + case1VisibilityBitmap := []byte{0x58, 0x80} case1VisibilityMap := map[int]bool{ 0: false, 1: true, @@ -970,6 +978,36 @@ func TestTableMapOptMetaVisibility(t *testing.T) { 10: false, } + /* + CREATE TABLE _visibility( + `col0` INT, + `col1` INT, + `col2` INT, + `col3` INT, + `col4` INT, + `col5` INT, + `col6` INT, + `col7` INT, + `col8` INT, + `col9` INT, + `col10` INT + ); + */ + case2VisibilityBitmap := []byte{0xff, 0xe0} + case2VisibilityMap := map[int]bool{ + 0: true, + 1: true, + 2: true, + 3: true, + 4: true, + 5: true, + 6: true, + 7: true, + 8: true, + 9: true, + 10: true, + } + // Invisible column and INVISIBLE keyword is available only on MySQL 8.0.23+ testcases := []struct { data []byte @@ -982,6 +1020,30 @@ func TestTableMapOptMetaVisibility(t *testing.T) { expectedVisibilityBitmap: case1VisibilityBitmap, expectedVisibilityMap: case1VisibilityMap, }, + { + // mysql 5.7, case2 + data: []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07"), + expectedVisibilityBitmap: []byte(nil), + expectedVisibilityMap: map[int]bool(nil), + }, + { + // mysql 8.0, case2 + data: []byte("^\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10\x0c\x02\xff\xe0"), + expectedVisibilityBitmap: case2VisibilityBitmap, + expectedVisibilityMap: case2VisibilityMap, + }, + { + // mariadb 10.4, case2 + data: []byte("\x12\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07"), + expectedVisibilityBitmap: []byte(nil), + expectedVisibilityMap: map[int]bool(nil), + }, + { + // mariadb 10.5, case2 + data: []byte("\x12\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10"), + expectedVisibilityBitmap: []byte(nil), + expectedVisibilityMap: map[int]bool(nil), + }, } for _, tc := range testcases {