From b20dc9cbe8b42d54b3c6c6c8b16221f1b6a200c6 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Thu, 3 Apr 2025 13:39:03 -0400 Subject: [PATCH 1/8] privileges and views --- tests/bwc/test_rolling_upgrade.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index c12509e5..d1fcc31b 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -1,5 +1,7 @@ import unittest from crate.client import connect +from crate.client.exceptions import ProgrammingError + from crate.qa.tests import NodeProvider, insert_data, wait_for_active_shards, UpgradePath ROLLING_UPGRADES_V4 = ( @@ -103,6 +105,7 @@ def _test_rolling_upgrade(self, path, nodes): ) CLUSTERED INTO {shards} SHARDS WITH (number_of_replicas={replicas}) ''') + c.execute("deny dql on table doc.t1 to arthur") c.execute("CREATE VIEW doc.v1 AS SELECT type, title, value FROM doc.t1") insert_data(conn, 'doc', 't1', 1000) @@ -148,8 +151,14 @@ def _test_rolling_upgrade(self, path, nodes): # Run a query as a user created on an older version (ensure user is read correctly from cluster state, auth works, etc) with connect(cluster.node().http_url, username='arthur', password='secret', error_trace=True) as custom_user_conn: c = custom_user_conn.cursor() - wait_for_active_shards(c, expected_active_shards) + # 'arthur' can only see 'parted' shards, 6 shards are for 't1' + wait_for_active_shards(c, expected_active_shards - 6) c.execute("SELECT 1") + # has no privilege + with self.assertRaisesRegex(ProgrammingError, "RelationUnknown.*"): + c.execute("EXPLAIN SELECT * FROM doc.t1") + # has privilege + c.execute("EXPLAIN SELECT * FROM doc.v1") cluster[idx] = new_node with connect(new_node.http_url, error_trace=True) as conn: @@ -159,8 +168,11 @@ def _test_rolling_upgrade(self, path, nodes): c.execute("select name from sys.users order by 1") self.assertEqual(c.fetchall(), [["arthur"], ["crate"]]) - c.execute("select * from sys.privileges") - self.assertEqual(c.fetchall(), [["CLUSTER", "arthur", "crate", None, "GRANT", "DQL"]]) + c.execute("select * from sys.privileges order by ident") + self.assertEqual( + c.fetchall(), + [['TABLE', 'arthur', 'crate', 'doc.t1', 'DENY', 'DQL'], + ['CLUSTER', 'arthur', 'crate', None, 'GRANT', 'DQL']]) c.execute(''' SELECT type, AVG(value) @@ -239,3 +251,14 @@ def _test_rolling_upgrade(self, path, nodes): ''') res = c.fetchone() self.assertEqual(res[0], nodes + 1) + + # Ensure Arthur can be dropped and re-added + c.execute("drop user arthur") + c.execute("select * from sys.privileges") + self.assertEqual(c.fetchall(), []) + + # Ensure view 'v' can be dropped and re-added + c.execute("DROP VIEW doc.v1") + c.execute("CREATE VIEW doc.v1 AS SELECT 11") + c.execute("SELECT * FROM doc.v1") + self.assertEqual(c.fetchall(), [[11]]) From 441a36a386afbda118611c97963a869e76b0a611 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Tue, 8 Apr 2025 19:05:07 -0400 Subject: [PATCH 2/8] default expr/gen col/constraints --- tests/bwc/test_rolling_upgrade.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index d1fcc31b..010e0235 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -108,12 +108,21 @@ def _test_rolling_upgrade(self, path, nodes): c.execute("deny dql on table doc.t1 to arthur") c.execute("CREATE VIEW doc.v1 AS SELECT type, title, value FROM doc.t1") insert_data(conn, 'doc', 't1', 1000) - c.execute("INSERT INTO doc.t1 (type, value, title, author) VALUES (1, 1, 'matchMe title', {name='no match name'})") c.execute("INSERT INTO doc.t1 (type, value, title, author) VALUES (2, 2, 'no match title', {name='matchMe name'})") - c.execute("INSERT INTO doc.t1 (title, author, o) VALUES ('prefix_check', {\"dyn_empty_array\" = []}, {\"dyn_ignored_subcol\" = 'hello'})") + c.execute(''' + create table doc.t2 ( + a int primary key, + b int not null, + c int default random() * 100, + d generated always as (a + b + c), + constraint d CHECK (d > a + b) + ) clustered into 1 shards with (number_of_replicas = 0) + ''') + expected_active_shards += 1 + c.execute(''' CREATE FUNCTION foo(INT) RETURNS INT @@ -237,6 +246,13 @@ def _test_rolling_upgrade(self, path, nodes): # Add the shards of the new partition primaries expected_active_shards += shards + c.execute("select count(*) from doc.t2") + count = c.fetchall()[0][0] + c.execute(f"insert into doc.t2(a, b) values ({idx}, {idx})") + c.execute("refresh table t2") + c.execute("select count(*) from doc.t2") + self.assertEqual(c.fetchall()[0][0], count + 1) + # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful with connect(cluster.node().http_url, error_trace=True) as conn: From 455cd05f43d7d84efcaf04405c0b08f38892c192 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Mon, 12 May 2025 19:44:46 -0400 Subject: [PATCH 3/8] fix --- tests/bwc/test_rolling_upgrade.py | 39 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index 010e0235..a680bc70 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -112,16 +112,17 @@ def _test_rolling_upgrade(self, path, nodes): c.execute("INSERT INTO doc.t1 (type, value, title, author) VALUES (2, 2, 'no match title', {name='matchMe name'})") c.execute("INSERT INTO doc.t1 (title, author, o) VALUES ('prefix_check', {\"dyn_empty_array\" = []}, {\"dyn_ignored_subcol\" = 'hello'})") - c.execute(''' - create table doc.t2 ( - a int primary key, - b int not null, - c int default random() * 100, - d generated always as (a + b + c), - constraint d CHECK (d > a + b) - ) clustered into 1 shards with (number_of_replicas = 0) - ''') - expected_active_shards += 1 + if int(path.from_version.split('.')[0]) >= 5: + c.execute(''' + create table doc.t2 ( + a int primary key, + b int not null, + c int default (random() + 1) * 100, + d generated always as (a + b + c), + constraint d CHECK (d > a + b) + ) clustered into 1 shards with (number_of_replicas = 0) + ''') + expected_active_shards += 1 c.execute(''' CREATE FUNCTION foo(INT) @@ -160,11 +161,10 @@ def _test_rolling_upgrade(self, path, nodes): # Run a query as a user created on an older version (ensure user is read correctly from cluster state, auth works, etc) with connect(cluster.node().http_url, username='arthur', password='secret', error_trace=True) as custom_user_conn: c = custom_user_conn.cursor() - # 'arthur' can only see 'parted' shards, 6 shards are for 't1' - wait_for_active_shards(c, expected_active_shards - 6) + wait_for_active_shards(c, expected_active_shards) c.execute("SELECT 1") # has no privilege - with self.assertRaisesRegex(ProgrammingError, "RelationUnknown.*"): + with self.assertRaises(ProgrammingError): c.execute("EXPLAIN SELECT * FROM doc.t1") # has privilege c.execute("EXPLAIN SELECT * FROM doc.v1") @@ -246,12 +246,13 @@ def _test_rolling_upgrade(self, path, nodes): # Add the shards of the new partition primaries expected_active_shards += shards - c.execute("select count(*) from doc.t2") - count = c.fetchall()[0][0] - c.execute(f"insert into doc.t2(a, b) values ({idx}, {idx})") - c.execute("refresh table t2") - c.execute("select count(*) from doc.t2") - self.assertEqual(c.fetchall()[0][0], count + 1) + if int(path.from_version.split('.')[0]) >= 5: + c.execute("select count(*) from doc.t2") + count = c.fetchall()[0][0] + c.execute(f"insert into doc.t2(a, b) values ({idx}, {idx})") + c.execute("refresh table t2") + c.execute("select count(*) from doc.t2") + self.assertEqual(c.fetchall()[0][0], count + 1) # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful From 850185d7172bbd0b6c313ec81f2c21ea71f1f5a3 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Mon, 12 May 2025 21:26:53 -0400 Subject: [PATCH 4/8] Arthur cannot see 't1' without dql starting from 5.6 --- tests/bwc/test_rolling_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index a680bc70..e489b537 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -161,7 +161,7 @@ def _test_rolling_upgrade(self, path, nodes): # Run a query as a user created on an older version (ensure user is read correctly from cluster state, auth works, etc) with connect(cluster.node().http_url, username='arthur', password='secret', error_trace=True) as custom_user_conn: c = custom_user_conn.cursor() - wait_for_active_shards(c, expected_active_shards) + wait_for_active_shards(c) c.execute("SELECT 1") # has no privilege with self.assertRaises(ProgrammingError): From ffbb2879e0d930903187e7bc01dcf0fb87a705a4 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Tue, 13 May 2025 17:41:46 -0400 Subject: [PATCH 5/8] Also test partitioned tables with constraint/gen-col/default-expr and the values --- tests/bwc/test_rolling_upgrade.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index e489b537..21d8b989 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -117,12 +117,21 @@ def _test_rolling_upgrade(self, path, nodes): create table doc.t2 ( a int primary key, b int not null, - c int default (random() + 1) * 100, + c int default (random() + 1), d generated always as (a + b + c), constraint d CHECK (d > a + b) ) clustered into 1 shards with (number_of_replicas = 0) ''') expected_active_shards += 1 + c.execute(''' + create table doc.t3 ( + a int primary key, + b int not null, + c int default (random() + 1), + d generated always as (a + b + c), + constraint d CHECK (d > a + b) + ) partitioned by (a) clustered into 1 shards with (number_of_replicas = 0) + ''') c.execute(''' CREATE FUNCTION foo(INT) @@ -247,12 +256,16 @@ def _test_rolling_upgrade(self, path, nodes): expected_active_shards += shards if int(path.from_version.split('.')[0]) >= 5: - c.execute("select count(*) from doc.t2") - count = c.fetchall()[0][0] c.execute(f"insert into doc.t2(a, b) values ({idx}, {idx})") c.execute("refresh table t2") - c.execute("select count(*) from doc.t2") - self.assertEqual(c.fetchall()[0][0], count + 1) + c.execute(f"select a, b, c>=1 and c<=2, d>a+b from doc.t2 where a = {idx}") + self.assertEqual(c.fetchall(), [[idx, idx, True, True]]) + if int(path.from_version.split('.')[1]) >= 4: + c.execute(f"insert into doc.t3(a, b) values ({idx}, {idx})") + expected_active_shards += 1 + c.execute("refresh table t3") + c.execute(f"select a, b, c>=1 and c<=2, d>a+b from doc.t3 where a = {idx}") + self.assertEqual(c.fetchall(), [[idx, idx, True, True]]) # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful From e39fc4999557cab79f82c17c8bb5af7b8e83ca4c Mon Sep 17 00:00:00 2001 From: jeeminso Date: Tue, 13 May 2025 20:12:53 -0400 Subject: [PATCH 6/8] Check table/partition versions per upgrade --- tests/bwc/test_rolling_upgrade.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index 21d8b989..c2c9103c 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -260,12 +260,19 @@ def _test_rolling_upgrade(self, path, nodes): c.execute("refresh table t2") c.execute(f"select a, b, c>=1 and c<=2, d>a+b from doc.t2 where a = {idx}") self.assertEqual(c.fetchall(), [[idx, idx, True, True]]) + old_version = '.'.join(map(str, node.version)) + c.execute("select distinct(version['created']) from information_schema.tables where table_name = 't2'") + self.assertEqual(c.fetchall(), [[old_version]]) if int(path.from_version.split('.')[1]) >= 4: c.execute(f"insert into doc.t3(a, b) values ({idx}, {idx})") expected_active_shards += 1 c.execute("refresh table t3") c.execute(f"select a, b, c>=1 and c<=2, d>a+b from doc.t3 where a = {idx}") self.assertEqual(c.fetchall(), [[idx, idx, True, True]]) + c.execute("select distinct(version['created']) from information_schema.tables where table_name = 't3'") + self.assertEqual(c.fetchall(), [[old_version]]) + c.execute("select distinct(version['created']) from information_schema.table_partitions where table_name = 't3'") + self.assertEqual(c.fetchall(), [[old_version]]) # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful From ec207bcf25309305fbd39116aeaf9d24c7c026bf Mon Sep 17 00:00:00 2001 From: jeeminso Date: Tue, 13 May 2025 21:14:19 -0400 Subject: [PATCH 7/8] Anoter behaviour change since 5.9; new partition versions follow latest version after completing the upgrade --- tests/bwc/test_rolling_upgrade.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index c2c9103c..034e957b 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -30,14 +30,6 @@ ) ROLLING_UPGRADES_V5 = ( - UpgradePath('5.0.x', '5.1.x'), - UpgradePath('5.1.x', '5.2.x'), - UpgradePath('5.2.x', '5.3.x'), - UpgradePath('5.3.x', '5.4.x'), - UpgradePath('5.4.x', '5.5.x'), - UpgradePath('5.5.x', '5.6.x'), - UpgradePath('5.6.x', '5.7.x'), - UpgradePath('5.7.x', '5.8.x'), UpgradePath('5.8.x', '5.9.x'), UpgradePath('5.9.x', '5.10.x'), UpgradePath('5.10.x', '5.10'), @@ -263,7 +255,7 @@ def _test_rolling_upgrade(self, path, nodes): old_version = '.'.join(map(str, node.version)) c.execute("select distinct(version['created']) from information_schema.tables where table_name = 't2'") self.assertEqual(c.fetchall(), [[old_version]]) - if int(path.from_version.split('.')[1]) >= 4: + if int(path.from_version.split('.')[1]) >= 9: c.execute(f"insert into doc.t3(a, b) values ({idx}, {idx})") expected_active_shards += 1 c.execute("refresh table t3") @@ -271,8 +263,12 @@ def _test_rolling_upgrade(self, path, nodes): self.assertEqual(c.fetchall(), [[idx, idx, True, True]]) c.execute("select distinct(version['created']) from information_schema.tables where table_name = 't3'") self.assertEqual(c.fetchall(), [[old_version]]) - c.execute("select distinct(version['created']) from information_schema.table_partitions where table_name = 't3'") - self.assertEqual(c.fetchall(), [[old_version]]) + partition_version = old_version + # after all 3 nodes are upgraded, newly created partition's version should follow the newer version + if idx == 2: + partition_version = '.'.join(map(str, new_node.version)) + c.execute(f"select version['created'] from information_schema.table_partitions where table_name = 't3' and values['a'] = {idx}") + self.assertEqual(c.fetchall(), [[partition_version]]) # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful From 5e8cc68661e764e2a733954c20e5823414dc82e1 Mon Sep 17 00:00:00 2001 From: jeeminso Date: Wed, 14 May 2025 15:37:04 -0400 Subject: [PATCH 8/8] Revert temporary changes --- tests/bwc/test_rolling_upgrade.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/bwc/test_rolling_upgrade.py b/tests/bwc/test_rolling_upgrade.py index 034e957b..373688d8 100644 --- a/tests/bwc/test_rolling_upgrade.py +++ b/tests/bwc/test_rolling_upgrade.py @@ -30,6 +30,14 @@ ) ROLLING_UPGRADES_V5 = ( + UpgradePath('5.0.x', '5.1.x'), + UpgradePath('5.1.x', '5.2.x'), + UpgradePath('5.2.x', '5.3.x'), + UpgradePath('5.3.x', '5.4.x'), + UpgradePath('5.4.x', '5.5.x'), + UpgradePath('5.5.x', '5.6.x'), + UpgradePath('5.6.x', '5.7.x'), + UpgradePath('5.7.x', '5.8.x'), UpgradePath('5.8.x', '5.9.x'), UpgradePath('5.9.x', '5.10.x'), UpgradePath('5.10.x', '5.10'),