Merge remote-tracking branch 'upstream/main' into add-default-role

Author: flaneur2020

.github/mergify.yml

@@ -13,7 +13,8 @@ queue_rules:
       - check-success=test_stateless_standalone_linux
       - check-success=test_stateless_cluster_linux
       - check-success=test_stateful_standalone_linux
-      - check-success~=^test_sqllogic_(base|ydb)_standalone_linux$
+      - check-success=test_sqllogic_base_standalone_linux
+      - check-success=test_sqllogic_ydb_standalone_linux
 
   - name: docs_queue
     conditions:

Cargo.lock

@@ -58,7 +58,8 @@ exclude = ["tools/fuzz"]
 [profile.release]
 debug = 1
 lto = "thin"
-codegen-units = 1 # Reduce number of codegen units to increase optimizations.
+overflow-checks = false
+codegen-units = 1       # Reduce number of codegen units to increase optimizations.
 
 [profile.release.package]
 arrow2 = { codegen-units = 4 }
@@ -67,9 +68,21 @@ databend-query = { codegen-units = 4 }
 
 [profile.bench]
 debug = true
+overflow-checks = false
 
 [profile.dev]
 split-debuginfo = "unpacked"
+overflow-checks = false
+
+[profile.test]
+opt-level = 0
+debug = 0
+codegen-units = 16
+lto = false
+incremental = true
+debug-assertions = true
+overflow-checks = true
+rpath = false
 
 [profile.dev.package]
 addr2line = { opt-level = 3 }

Cargo.toml

@@ -19,6 +19,7 @@ mod explain;
 mod insert;
 mod kill;
 mod presign;
+mod share;
 mod show;
 mod stage;
 mod statement;
@@ -33,6 +34,7 @@ pub use explain::*;
 pub use insert::*;
 pub use kill::*;
 pub use presign::*;
+pub use share::*;
 pub use show::*;
 pub use stage::*;
 pub use statement::*;

common/ast/src/ast/statements/mod.rs

@@ -0,0 +1,39 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use crate::ast::Identifier;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreateShareStmt<'a> {
+    pub if_not_exists: bool,
+    pub share: Identifier<'a>,
+    pub comment: Option<String>,
+}
+
+impl Display for CreateShareStmt<'_> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "CREATE SHARE ")?;
+        if self.if_not_exists {
+            write!(f, "IF NOT EXISTS ")?;
+        }
+        write!(f, "{}", self.share)?;
+        if let Some(comment) = &self.comment {
+            write!(f, " COMMENT = '{comment}'")?;
+        }
+        Ok(())
+    }
+}

common/ast/src/ast/statements/share.rs

@@ -157,6 +157,9 @@ pub enum Statement<'a> {
     },
 
     Presign(PresignStmt),
+
+    // share
+    CreateShare(CreateShareStmt<'a>),
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -358,6 +361,7 @@ impl<'a> Display for Statement<'a> {
             Statement::DescribeStage { stage_name } => write!(f, "DESC STAGE {stage_name}")?,
             Statement::Call(stmt) => write!(f, "{stmt}")?,
             Statement::Presign(stmt) => write!(f, "{stmt}")?,
+            Statement::CreateShare(stmt) => write!(f, "{stmt}")?,
         }
         Ok(())
     }

common/ast/src/ast/statements/statement.rs

@@ -208,7 +208,7 @@ pub fn statement(i: Input) -> IResult<StatementMsg> {
     );
     let show_tables = map(
         rule! {
-            SHOW ~ FULL? ~ TABLES ~ HISTORY? ~ ( FROM ~ ^#ident )? ~ #show_limit?
+            SHOW ~ FULL? ~ TABLES ~ HISTORY? ~ ( ( FROM | IN ) ~ ^#ident )? ~ #show_limit?
         },
         |(_, opt_full, _, opt_history, opt_database, limit)| {
             Statement::ShowTables(ShowTablesStmt {
@@ -741,6 +741,23 @@ pub fn statement(i: Input) -> IResult<StatementMsg> {
         },
     );
 
+    // share statements
+    let create_share = map(
+        rule! {
+            CREATE ~ SHARE ~ (IF ~ NOT ~ EXISTS )? ~ #ident ~ ( COMMENT ~ "=" ~ #literal_string)?
+        },
+        |(_, _, opt_if_not_exists, share, comment_opt)| {
+            Statement::CreateShare(CreateShareStmt {
+                if_not_exists: opt_if_not_exists.is_some(),
+                share,
+                comment: match comment_opt {
+                    Some(opt) => Some(opt.2),
+                    None => None,
+                },
+            })
+        },
+    );
+
     let statement_body = alt((
         rule!(
             #map(query, |query| Statement::Query(Box::new(query)))
@@ -826,6 +843,10 @@ pub fn statement(i: Input) -> IResult<StatementMsg> {
         rule!(
             #presign: "`PRESIGN [{DOWNLOAD | UPLOAD}] <location> [EXPIRE = 3600]`"
         ),
+        // share
+        rule!(
+            #create_share: "`CREATE SHARE <share_name> [ COMMENT = '<string_literal>' ]`"
+        ),
     ));
 
     map(

common/ast/src/parser/statement.rs

@@ -607,6 +607,8 @@ pub enum TokenKind {
     UPDATE,
     #[token("UPLOAD", ignore(ascii_case))]
     UPLOAD,
+    #[token("SHARE", ignore(ascii_case))]
+    SHARE,
     #[token("SUPER", ignore(ascii_case))]
     SUPER,
     #[token("STATUS", ignore(ascii_case))]

common/ast/src/parser/token.rs

@@ -142,6 +142,7 @@ fn test_statement() {
         r#"ALTER DATABASE catalog.c RENAME TO a;"#,
         r#"CREATE TABLE t (a INT COMMENT 'col comment') COMMENT='table comment';"#,
         r#"GRANT SELECT, CREATE ON * TO 'test-grant'@'localhost';"#,
+        r#"GRANT SELECT, CREATE ON *.* TO 'test-grant'@'localhost';"#,
         r#"GRANT SELECT, CREATE ON * TO USER 'test-grant'@'localhost';"#,
         r#"GRANT SELECT, CREATE ON * TO ROLE 'role1';"#,
         r#"GRANT ALL ON *.* TO 'test-grant'@'localhost';"#,
@@ -255,6 +256,8 @@ fn test_statement() {
         r#"PRESIGN @my_stage/path/to/file"#,
         r#"PRESIGN DOWNLOAD @my_stage/path/to/file"#,
         r#"PRESIGN UPLOAD @my_stage/path/to/file EXPIRE=7200"#,
+        r#"CREATE SHARE t COMMENT='share comment';"#,
+        r#"CREATE SHARE IF NOT EXISTS t;"#,
     ];
 
     for case in cases {

common/ast/tests/it/parser.rs

@@ -4938,6 +4938,30 @@ Grant(
 )
 
 
+---------- Input ----------
+GRANT SELECT, CREATE ON *.* TO 'test-grant'@'localhost';
+---------- Output ---------
+GRANT SELECT, CREATE ON *.* TO USER 'test-grant'@'localhost'
+---------- AST ------------
+Grant(
+    GrantStmt {
+        source: Privs {
+            privileges: [
+                Select,
+                Create,
+            ],
+            level: Global,
+        },
+        principal: User(
+            UserIdentity {
+                username: "test-grant",
+                hostname: "localhost",
+            },
+        ),
+    },
+)
+
+
 ---------- Input ----------
 GRANT SELECT, CREATE ON * TO USER 'test-grant'@'localhost';
 ---------- Output ---------
@@ -5883,3 +5907,41 @@ Presign(
 )
 
 
+---------- Input ----------
+CREATE SHARE t COMMENT='share comment';
+---------- Output ---------
+CREATE SHARE t COMMENT = 'share comment'
+---------- AST ------------
+CreateShare(
+    CreateShareStmt {
+        if_not_exists: false,
+        share: Identifier {
+            name: "t",
+            quote: None,
+            span: Ident(13..14),
+        },
+        comment: Some(
+            "share comment",
+        ),
+    },
+)
+
+
+---------- Input ----------
+CREATE SHARE IF NOT EXISTS t;
+---------- Output ---------
+CREATE SHARE IF NOT EXISTS t
+---------- AST ------------
+CreateShare(
+    CreateShareStmt {
+        if_not_exists: true,
+        share: Identifier {
+            name: "t",
+            quote: None,
+            span: Ident(27..28),
+        },
+        comment: None,
+    },
+)
+
+

common/ast/tests/it/testdata/statement.txt

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::borrow::Borrow;
+use std::sync::Arc;
 
 use super::Meter;
 pub struct BytesMeter;
@@ -24,3 +25,11 @@ impl<K> Meter<K, Vec<u8>> for BytesMeter {
         v.len()
     }
 }
+
+impl<K> Meter<K, Arc<Vec<u8>>> for BytesMeter {
+    type Measure = usize;
+    fn measure<Q: ?Sized>(&self, _: &Q, v: &Arc<Vec<u8>>) -> usize
+    where K: Borrow<Q> {
+        v.len()
+    }
+}

common/cache/src/meter/bytes_meter.rs

@@ -23,7 +23,6 @@ common-meta-types = { path = "../meta/types" }
 common-pipeline = { path = "../pipeline" }
 common-planners = { path = "../planners" }
 common-settings = { path = "../settings" }
-common-streams = { path = "../streams" }
 common-users = { path = "../users" }
 
 async-trait = "0.1.56"

common/catalog/Cargo.toml

@@ -34,7 +34,6 @@ use common_planners::Partitions;
 use common_planners::ReadDataSourcePlan;
 use common_planners::Statistics;
 use common_planners::TruncateTablePlan;
-use common_streams::SendableDataBlockStream;
 
 use crate::table_context::TableContext;
 
@@ -128,20 +127,16 @@ pub trait Table: Sync + Send {
         _: &ReadDataSourcePlan,
         _: &mut Pipeline,
     ) -> Result<()> {
-        unimplemented!()
+        Err(ErrorCode::UnImplement(format!(
+            "read2 operation for table {} is not implemented, table engine is {}",
+            self.name(),
+            self.get_table_info().meta.engine
+        )))
     }
 
     fn append2(&self, _: Arc<dyn TableContext>, _: &mut Pipeline) -> Result<()> {
-        unimplemented!()
-    }
-
-    async fn append_data(
-        &self,
-        _ctx: Arc<dyn TableContext>,
-        _stream: SendableDataBlockStream,
-    ) -> Result<SendableDataBlockStream> {
         Err(ErrorCode::UnImplement(format!(
-            "append operation for table {} is not implemented, table engine is {}",
+            "append2 operation for table {} is not implemented, table engine is {}",
             self.name(),
             self.get_table_info().meta.engine
         )))
@@ -216,5 +211,5 @@ pub struct TableStatistics {
     pub num_rows: Option<u64>,
     pub data_size: Option<u64>,
     pub data_size_compressed: Option<u64>,
-    pub index_length: Option<u64>,
+    pub index_size: Option<u64>,
 }

common/catalog/src/table.rs

@@ -22,22 +22,23 @@ use common_arrow::parquet::encoding::Encoding;
 use common_arrow::parquet::metadata::ThriftFileMetaData;
 use common_arrow::parquet::write::Version;
 use common_arrow::write_parquet_file;
-use common_datavalues::DataSchemaRef;
+use common_datavalues::DataSchema;
 use common_exception::ErrorCode;
 use common_exception::Result;
 
 use crate::DataBlock;
 
-pub fn serialize_data_blocks(
+pub fn serialize_data_blocks_with_compression(
     blocks: Vec<DataBlock>,
-    schema: &DataSchemaRef,
+    schema: impl AsRef<DataSchema>,
     buf: &mut Vec<u8>,
+    compression: CompressionOptions,
 ) -> Result<(u64, ThriftFileMetaData)> {
-    let arrow_schema = schema.to_arrow();
+    let arrow_schema = schema.as_ref().to_arrow();
 
     let row_group_write_options = WriteOptions {
         write_statistics: false,
-        compression: CompressionOptions::Lz4Raw,
+        compression,
         version: Version::V2,
     };
     let batches = blocks
@@ -75,6 +76,14 @@ pub fn serialize_data_blocks(
     }
 }
 
+pub fn serialize_data_blocks(
+    blocks: Vec<DataBlock>,
+    schema: impl AsRef<DataSchema>,
+    buf: &mut Vec<u8>,
+) -> Result<(u64, ThriftFileMetaData)> {
+    serialize_data_blocks_with_compression(blocks, schema, buf, CompressionOptions::Lz4Raw)
+}
+
 fn col_encoding(_data_type: &ArrowDataType) -> Encoding {
     // Although encoding does work, parquet2 has not implemented decoding of DeltaLengthByteArray yet, we fallback to Plain
     // From parquet2: Decoding "DeltaLengthByteArray"-encoded required V2 pages is not yet implemented for Binary.

common/datablocks/src/utils.rs

@@ -9,6 +9,7 @@ edition = "2021"
 [features]
 
 [dependencies]
+common-arrow = { path = "../arrow" }
 common-base = { path = "../base" }
 common-cache = { path = "../cache" }
 common-config = { path = "../config" }