Skip to content

Commit 27d7c81

Browse files
flaneur2020flaneur2020
committed
merge upstream/main
2 parents 29ce38d + 97e1bb2 commit 27d7c81

File tree

625 files changed

+12014
-16930
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

625 files changed

+12014
-16930
lines changed

‎.cargo/audit.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
[advisories]
22
ignore = [
3+
# rio: v0.9.4 https://rustsec.org/advisories/RUSTSEC-2020-0021
4+
# allows a use-after-free buffer access when a future is leaked
5+
"RUSTSEC-2020-0021",
36
# time: Potential segfault in the time crate
47
# We are not accected by this CVE.
58
# And there is no actions we can take, waiting for upstream.

‎.github/actions/test_sqllogic_standalone_linux/action.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ inputs:
99
description: ""
1010
required: true
1111
default: "x86_64-unknown-linux-gnu"
12+
dirs:
13+
description: "logic test suites dirs"
14+
required: true
15+
default: ""
1216
runs:
1317
using: "composite"
1418
steps:
@@ -27,7 +31,7 @@ runs:
2731
- name: Run sqllogic Tests with Standalone mode with embedded meta-store
2832
shell: bash
2933
run: |
30-
bash ./scripts/ci/ci-run-sqllogic-tests.sh
34+
bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
3135
3236
# - name: Upload failure
3337
# if: failure()

‎.github/actions/test_stateful_hive_standalone/action.yml

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,20 @@ runs:
4949
shell: bash
5050
run: |
5151
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'CREATE TABLE t_1 (id string, t_bool boolean, t_float float, t_double double, t_data date, t_array array<int>, t_array2 array<string>) stored as parquet;'"
52-
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/t_1.parquet\" OVERWRITE into table t_1;'"
53-
mkdir -p .databend/stateless_test_data/user/hive/warehouse/t_1/
54-
cp tests/data/hive/t_1.parquet .databend/stateless_test_data/user/hive/warehouse/t_1/
52+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/t_1/t_1.parquet\" OVERWRITE into table t_1;'"
53+
mkdir -p .databend/stateless_test_data/user/hive/warehouse/
54+
cp -r tests/data/hive/t_1 .databend/stateless_test_data/user/hive/warehouse/
55+
56+
- name: Hive Create Partition Table&Load Data
57+
shell: bash
58+
run: |
59+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'CREATE TABLE if not exists customer_p2 ( foo VARCHAR(25) , c_region2 VARCHAR(25) , c_nation2 VARCHAR(10)) partitioned by (c_region VARCHAR(12), c_nation VARCHAR(15)) stored as parquet;'"
60+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/customer_p2/c_region=ASIA/c_nation=CHINA\" OVERWRITE into table customer_p2 partition(c_region = \"ASIA\", c_nation = \"CHINA\");'"
61+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/customer_p2/c_region=ASIA/c_nation=JAPAN\" OVERWRITE into table customer_p2 partition(c_region = \"ASIA\", c_nation = \"JAPAN\");'"
62+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/customer_p2/c_region=EUROPE/c_nation=FRANCE\" OVERWRITE into table customer_p2 partition(c_region = \"EUROPE\", c_nation = \"FRANCE\");'"
63+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/customer_p2/c_region=EUROPE/c_nation=RUSSIA\" OVERWRITE into table customer_p2 partition(c_region = \"EUROPE\", c_nation = \"RUSSIA\");'"
64+
docker-compose -f "./docker/it-hive/hive-docker-compose.yml" exec -T hive-server bash -c "/opt/hive/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -e 'load data local inpath \"/databend-data/customer_p2/c_region=EUROPE/c_nation=GERMANY\" OVERWRITE into table customer_p2 partition(c_region = \"EUROPE\", c_nation = \"GERMANY\");'"
65+
cp -r tests/data/hive/customer_p2 .databend/stateless_test_data/user/hive/warehouse/
5566
5667
- name: Run Stateful Tests with Standalone mode
5768
shell: bash

‎.github/actions/test_stateful_standalone_linux/action.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,15 @@ runs:
4141
- name: Run Stateful Tests with Standalone mode (ubuntu-latest only)
4242
shell: bash
4343
run: |
44-
bash ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh
44+
./scripts/ci/ci-run-stateful-tests-standalone-s3.sh
45+
46+
- name: Upload .stdout files
47+
if: failure()
48+
uses: actions/upload-artifact@v3
49+
with:
50+
name: test-stateful-standalone-linux-stdouts
51+
path: |
52+
./tests/suites/1_stateful/*/*.stdout
4553
4654
- name: Upload failure
4755
if: failure()

‎.github/mergify.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ queue_rules:
1313
- check-success=test_stateless_standalone_linux
1414
- check-success=test_stateless_cluster_linux
1515
- check-success=test_stateful_standalone_linux
16-
- check-success=test_sqllogic_standalone_linux
16+
- check-success~=^test_sqllogic_(base|ydb)_standalone_linux$
1717

1818
- name: docs_queue
1919
conditions:

‎.github/workflows/dev-linux.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,20 @@ jobs:
121121
- uses: ./.github/actions/test_stateless_cluster_linux
122122

123123
test_sqllogic_standalone_linux:
124-
timeout-minutes: 20
124+
timeout-minutes: 15
125+
name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
125126
runs-on: [self-hosted, X64, Linux, development]
126127
needs: build_gnu
128+
strategy:
129+
matrix:
130+
dirs:
131+
- "base"
132+
- "ydb"
127133
steps:
128134
- uses: actions/checkout@v3
129135
- uses: ./.github/actions/test_sqllogic_standalone_linux
136+
with:
137+
dirs: ${{ matrix.dirs }}
130138

131139
test_stateful_standalone_linux:
132140
timeout-minutes: 10

‎Cargo.lock

Lines changed: 33 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎common/ast/src/parser/expr.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1172,7 +1172,7 @@ pub fn type_name(i: Input) -> IResult<TypeName> {
11721172
);
11731173
let ty_string = value(
11741174
TypeName::String,
1175-
rule! { ( STRING | VARCHAR | CHAR ) ~ ( "(" ~ #literal_u64 ~ ")" )? },
1175+
rule! { ( STRING | VARCHAR | CHAR | CHARACTER | TEXT ) ~ ( "(" ~ #literal_u64 ~ ")" )? },
11761176
);
11771177
let ty_object = value(TypeName::Object, rule! { OBJECT | MAP });
11781178
let ty_variant = value(TypeName::Variant, rule! { VARIANT | JSON });

‎common/ast/src/parser/token.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,8 @@ pub enum TokenKind {
293293
CONNECTION,
294294
#[token("CHAR", ignore(ascii_case))]
295295
CHAR,
296+
#[token("CHARACTER", ignore(ascii_case))]
297+
CHARACTER,
296298
#[token("COPY_OPTIONS", ignore(ascii_case))]
297299
COPY_OPTIONS,
298300
#[token("COPY", ignore(ascii_case))]
@@ -617,6 +619,8 @@ pub enum TokenKind {
617619
TABLE,
618620
#[token("TABLES", ignore(ascii_case))]
619621
TABLES,
622+
#[token("TEXT", ignore(ascii_case))]
623+
TEXT,
620624
#[token("TENANTSETTING", ignore(ascii_case))]
621625
TENANTSETTING,
622626
#[token("THEN", ignore(ascii_case))]
@@ -881,7 +885,7 @@ impl TokenKind {
881885
| TokenKind::ARRAY
882886
| TokenKind::AS
883887
// | TokenKind::CHAR
884-
// | TokenKind::CHARACTER
888+
| TokenKind::CHARACTER
885889
| TokenKind::CREATE
886890
| TokenKind::EXCEPT
887891
// | TokenKind::FETCH

‎common/ast/tests/it/testdata/expr-error.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ error:
3131
1 | CAST(col1 AS foo)
3232
| ---- ^^^
3333
| | |
34-
| | expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 27 more ...
34+
| | expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 29 more ...
3535
| | while parsing type name
3636
| while parsing `CAST(... AS ...)`
3737
| while parsing expression

‎common/ast/tests/it/testdata/statement-error.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ error:
3131
1 | create table a (c varch)
3232
| ------ - ^^^^^
3333
| | | |
34-
| | | expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 27 more ...
34+
| | | expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 29 more ...
3535
| | | while parsing type name
3636
| | while parsing `<column name> <type> [DEFAULT <default value>] [COMMENT '<comment>']`
3737
| while parsing `CREATE TABLE [IF NOT EXISTS] [<database>.]<table> [<source>] [<table_options>]`
@@ -44,7 +44,7 @@ error:
4444
--> SQL:1:25
4545
|
4646
1 | create table a (c tuple())
47-
| ------ - ----- ^ expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 29 more ...
47+
| ------ - ----- ^ expected `BOOLEAN`, `BOOL`, `UINT8`, `TINYINT`, `UINT16`, `SMALLINT`, or 31 more ...
4848
| | | |
4949
| | | while parsing type name
5050
| | while parsing `<column name> <type> [DEFAULT <default value>] [COMMENT '<comment>']`

‎common/catalog/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,6 @@ common-users = { path = "../users" }
2828

2929
async-trait = "0.1.56"
3030
dyn-clone = "1.0.6"
31-
opendal = { version = "0.11.3", features = ["retry"] }
31+
opendal = { version = "0.11.4", features = ["retry"] }
3232
parking_lot = "0.12.1"
3333
time = "0.3.10"

‎common/catalog/src/catalog.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
use std::any::Any;
1516
use std::collections::HashMap;
1617
use std::sync::Arc;
1718

@@ -162,6 +163,8 @@ pub trait Catalog: DynClone + Send + Sync {
162163
))
163164
}
164165

166+
fn as_any(&self) -> &dyn Any;
167+
165168
// Get table engines
166169
fn get_table_engines(&self) -> Vec<StorageDescription> {
167170
unimplemented!()

‎common/config/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ common-tracing = { path = "../tracing" }
1919
async-trait = "0.1.56"
2020
clap = { version = "3.2.5", features = ["derive", "env"] }
2121
once_cell = "1.12.0"
22-
opendal = { version = "0.11.3", features = ["retry", "compress"] }
22+
opendal = { version = "0.11.4", features = ["retry", "compress"] }
2323
semver = "1.0.10"
2424
serde = { version = "1.0.137", features = ["derive"] }
2525
serfig = "0.0.2"

‎common/contexts/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ test = false
1212
common-base = { path = "../base" }
1313

1414
async-trait = "0.1.56"
15-
opendal = { version = "0.11.3", features = ["retry"] }
15+
opendal = { version = "0.11.4", features = ["retry"] }
1616
time = "0.3.10"

‎common/datablocks/tests/it/kernels/data_block_take.rs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
use common_arrow::arrow::bitmap::MutableBitmap;
1516
use common_datablocks::*;
1617
use common_datavalues::prelude::*;
1718
use common_exception::Result;
@@ -43,3 +44,36 @@ fn test_data_block_take() -> Result<()> {
4344

4445
Ok(())
4546
}
47+
48+
#[test]
49+
fn test_data_block_take_with_nullable() -> Result<()> {
50+
let schema = DataSchemaRefExt::create(vec![
51+
DataField::new("a", NullableType::new_impl(i64::to_data_type())),
52+
DataField::new("b", Vu8::to_data_type()),
53+
]);
54+
55+
let mut validity = MutableBitmap::new();
56+
validity.extend_constant(3, false);
57+
58+
let col1 =
59+
NullableColumn::wrap_inner(Series::from_data(vec![1i64, 2, 1]), Some(validity.into()));
60+
let raw = DataBlock::create(schema, vec![
61+
col1,
62+
Series::from_data(vec!["b1", "b2", "b3"]),
63+
]);
64+
65+
let take = DataBlock::block_take_by_indices(&raw, &[0, 2])?;
66+
assert_eq!(raw.schema(), take.schema());
67+
68+
let expected = vec![
69+
"+------+----+",
70+
"| a | b |",
71+
"+------+----+",
72+
"| NULL | b1 |",
73+
"| NULL | b3 |",
74+
"+------+----+",
75+
];
76+
common_datablocks::assert_blocks_eq(expected, &[take]);
77+
78+
Ok(())
79+
}

‎common/exception/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ test = false
1414
common-arrow = { path = "../arrow" }
1515

1616
anyhow = "1.0.58"
17+
bincode = { version = "2.0.0-rc.1", features = ["serde", "std", "alloc"] }
1718
octocrab = "0.16.0"
1819
paste = "1.0.7"
1920
prost = "0.10.4"
@@ -24,5 +25,4 @@ time = "0.3.10"
2425
tonic = "0.7.2"
2526

2627
# Github dependencies
27-
bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" }
2828
sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" }

‎common/exception/src/exception_into.rs

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,40 @@ impl From<common_arrow::arrow::error::Error> for ErrorCode {
9494
}
9595
}
9696

97-
impl From<Box<bincode::ErrorKind>> for ErrorCode {
98-
fn from(error: Box<bincode::ErrorKind>) -> Self {
97+
impl From<bincode::error::EncodeError> for ErrorCode {
98+
fn from(error: bincode::error::EncodeError) -> Self {
9999
ErrorCode::from_std_error(error)
100100
}
101101
}
102102

103+
impl From<bincode::error::DecodeError> for ErrorCode {
104+
fn from(error: bincode::error::DecodeError) -> Self {
105+
ErrorCode::from_std_error(error)
106+
}
107+
}
108+
109+
impl From<bincode::serde::EncodeError> for ErrorCode {
110+
fn from(error: bincode::serde::EncodeError) -> Self {
111+
ErrorCode::create(
112+
1002,
113+
format!("{error:?}"),
114+
None,
115+
Some(ErrorCodeBacktrace::Origin(Arc::new(Backtrace::capture()))),
116+
)
117+
}
118+
}
119+
120+
impl From<bincode::serde::DecodeError> for ErrorCode {
121+
fn from(error: bincode::serde::DecodeError) -> Self {
122+
ErrorCode::create(
123+
1002,
124+
format!("{error:?}"),
125+
None,
126+
Some(ErrorCodeBacktrace::Origin(Arc::new(Backtrace::capture()))),
127+
)
128+
}
129+
}
130+
103131
impl From<serde_json::Error> for ErrorCode {
104132
fn from(error: serde_json::Error) -> Self {
105133
ErrorCode::from_std_error(error)

0 commit comments

Comments
 (0)