From a1ed5c51b1c2fb7df3f01bda085e8b77155722cc Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 14:25:29 +0200 Subject: [PATCH 01/64] Feature: function about convert_timezone #16177 --- src/query/expression/src/utils/date_helper.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index 175d63e185e97..ad27540e8e9e2 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -243,6 +243,7 @@ impl TzLUT { pub trait DateConverter { fn to_date(&self, tz: Tz) -> NaiveDate; fn to_timestamp(&self, tz: Tz) -> DateTime; + fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option>) -> DateTime; } impl DateConverter for T @@ -264,6 +265,31 @@ where T: AsPrimitive } tz.timestamp_opt(secs, nanos as u32).unwrap() } + /// Convert a timestamp to the specify timezone. + /// + /// # Parameters + /// - `target_tz`: Timezone in which the timestamp will be converted + /// - `src_timestamp`: Source timestamp to be converted + /// - `src_tz`: Timezone of the timestamp to be converted - Optional + /// - `src_ntz_timestamp`: Source timestamp with unspecified timezone to be converted - Optional + fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option) -> DateTime { + + let timestamp_to_convert: DateTime; + if let (Some(ntz_timestamp), Some(tz)) = (src_ntz_timestamp, src_tz) { + timestamp_to_convert = tz.from_local_datetime(&ntz_timestamp).unwrap(); + } + else { + timestamp_to_convert = src_timestamp; + } + + if timestamp_to_convert.timezone() != target_tz { + timestamp_to_convert.with_timezone(&target_tz) + } + else { + timestamp_to_convert + } + + } } pub const MICROSECS_PER_DAY: i64 = 86_400_000_000; From e5816afd7aabed75de7fa96656b5d6eb62190886 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 14:54:53 +0200 Subject: [PATCH 02/64] Correction trait type --- src/query/expression/src/utils/date_helper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index ad27540e8e9e2..a998c80aab144 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -243,7 +243,7 @@ impl TzLUT { pub trait DateConverter { fn to_date(&self, tz: Tz) -> NaiveDate; fn to_timestamp(&self, tz: Tz) -> DateTime; - fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option>) -> DateTime; + fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option) -> DateTime; } impl DateConverter for T From 8112be044a8df37991fb95bd41f1a211b416f13a Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 15:07:33 +0200 Subject: [PATCH 03/64] Formatting the function --- src/query/expression/src/utils/date_helper.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index a998c80aab144..55d82b1a5f641 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -272,23 +272,24 @@ where T: AsPrimitive /// - `src_timestamp`: Source timestamp to be converted /// - `src_tz`: Timezone of the timestamp to be converted - Optional /// - `src_ntz_timestamp`: Source timestamp with unspecified timezone to be converted - Optional - fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option) -> DateTime { - + fn convert_timezone( + target_tz: Tz, + src_timestamp: DateTime, + src_tz: Option, + src_ntz_timestamp: Option, + ) -> DateTime { let timestamp_to_convert: DateTime; if let (Some(ntz_timestamp), Some(tz)) = (src_ntz_timestamp, src_tz) { timestamp_to_convert = tz.from_local_datetime(&ntz_timestamp).unwrap(); - } - else { + } else { timestamp_to_convert = src_timestamp; } if timestamp_to_convert.timezone() != target_tz { timestamp_to_convert.with_timezone(&target_tz) - } - else { + } else { timestamp_to_convert } - } } From b61543f19a8be67ccc7bc4d8f2467db11e275d5c Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 15:13:42 +0200 Subject: [PATCH 04/64] Solving compiling issue --- src/query/expression/src/utils/date_helper.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index 55d82b1a5f641..d38124749fdf7 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -273,6 +273,7 @@ where T: AsPrimitive /// - `src_tz`: Timezone of the timestamp to be converted - Optional /// - `src_ntz_timestamp`: Source timestamp with unspecified timezone to be converted - Optional fn convert_timezone( + &self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, From 3fe6c575b7f97e5e16ec43df7d54c6be94bd44f2 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 15:20:16 +0200 Subject: [PATCH 05/64] Correcting file format --- src/query/expression/src/utils/date_helper.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index d38124749fdf7..bfc41a8ffaa66 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -243,7 +243,13 @@ impl TzLUT { pub trait DateConverter { fn to_date(&self, tz: Tz) -> NaiveDate; fn to_timestamp(&self, tz: Tz) -> DateTime; - fn convert_timezone(&self, target_tz: Tz, src_timestamp: DateTime, src_tz: Option, src_ntz_timestamp: Option) -> DateTime; + fn convert_timezone( + &self, + target_tz: Tz, + src_timestamp: DateTime, + src_tz: Option, + src_ntz_timestamp: Option, + ) -> DateTime; } impl DateConverter for T From dea89c7625a5d1910b9a35c206cde64aa0a1a8b4 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 17:49:46 +0200 Subject: [PATCH 06/64] adding tests --- src/query/functions/tests/it/scalars/datetime.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 783dc6b1f0ef8..e390fc7893ca7 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -15,6 +15,7 @@ use std::io::Write; use databend_common_expression::types::*; +use databend_common_expression::Column::Null; use databend_common_expression::FromData; use goldenfile::Mint; @@ -63,6 +64,15 @@ fn test_to_timestamp(file: &mut impl Write) { )]); } +fn test_convert_timezone(file: &mut impl Write) { + run_ast(file, "convert_timezone(a,b,c,d)", &[ + ("a", DateType::from_data(vec![-100, 0, 100])), + ("b", TimestampType::from_data(vec![12321231])), + ("c", Null::from_data()), + ("d", Null::from_data()), + ]); +} + fn test_to_date(file: &mut impl Write) { run_ast(file, "to_date(-354286)", &[]); run_ast(file, "to_date(-354285)", &[]); From 4d5b0c9d45cc44048d20d63bbf88c49c263dbf02 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 18:22:31 +0200 Subject: [PATCH 07/64] Correction unti test --- src/query/functions/tests/it/scalars/datetime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index e390fc7893ca7..4e953e58597eb 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -68,8 +68,8 @@ fn test_convert_timezone(file: &mut impl Write) { run_ast(file, "convert_timezone(a,b,c,d)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), ("b", TimestampType::from_data(vec![12321231])), - ("c", Null::from_data()), - ("d", Null::from_data()), + ("c", None::Datetype), + ("d", None::TimestampType), ]); } From 1aff0e82a77ba5166d0606d97a8e9ebfaac59df9 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 18:30:17 +0200 Subject: [PATCH 08/64] Correction unit tests 2 --- src/query/functions/tests/it/scalars/datetime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 4e953e58597eb..cca49a38346a8 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -68,8 +68,8 @@ fn test_convert_timezone(file: &mut impl Write) { run_ast(file, "convert_timezone(a,b,c,d)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), ("b", TimestampType::from_data(vec![12321231])), - ("c", None::Datetype), - ("d", None::TimestampType), + ("c", DateType::from_data(vec![])), + ("d", TimestampType::from_data(vec![])), ]); } From c3762003b016621af7f117dde9c9227cb8810036 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 18:37:02 +0200 Subject: [PATCH 09/64] Correction unit tests 3 --- src/query/functions/tests/it/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index cca49a38346a8..abc65fc87dffa 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -15,7 +15,6 @@ use std::io::Write; use databend_common_expression::types::*; -use databend_common_expression::Column::Null; use databend_common_expression::FromData; use goldenfile::Mint; @@ -37,6 +36,7 @@ fn test_datetime() { test_to_number(file); test_rounder_functions(file); test_date_date_diff(file); + test_convert_timezone(file); } fn test_to_timestamp(file: &mut impl Write) { From af370a69996ac4341223e588b4e3d6d6821c9936 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 18:49:44 +0200 Subject: [PATCH 10/64] Correction unit tests 4 --- src/query/functions/tests/it/scalars/datetime.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index abc65fc87dffa..9059bd1642daf 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -67,10 +67,17 @@ fn test_to_timestamp(file: &mut impl Write) { fn test_convert_timezone(file: &mut impl Write) { run_ast(file, "convert_timezone(a,b,c,d)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", TimestampType::from_data(vec![12321231])), + ("b", TimestampType::from_data(vec![315360000000])), ("c", DateType::from_data(vec![])), ("d", TimestampType::from_data(vec![])), ]); + + run_ast(file, "convert_timezone(a,b,c,d)", &[ + ("a", DateType::from_data(vec![-100, 0, 100])), + ("b", TimestampType::from_data(vec![315360000000])), + ("c", DateType::from_data(vec![-50, 0, 50])), + ("d", TimestampType::from_data(vec![215360000000])), + ]); } fn test_to_date(file: &mut impl Write) { From 1074c0a3f3663ba03781f9c2d7873112298b9e0c Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 19:28:49 +0200 Subject: [PATCH 11/64] Correction unit tests 5 --- src/query/functions/tests/it/scalars/datetime.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 9059bd1642daf..3826f2a0d6312 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -67,16 +67,16 @@ fn test_to_timestamp(file: &mut impl Write) { fn test_convert_timezone(file: &mut impl Write) { run_ast(file, "convert_timezone(a,b,c,d)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", TimestampType::from_data(vec![315360000000])), + ("b", TimestampType::from_data(vec![31536000])), ("c", DateType::from_data(vec![])), ("d", TimestampType::from_data(vec![])), ]); run_ast(file, "convert_timezone(a,b,c,d)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", TimestampType::from_data(vec![315360000000])), + ("b", TimestampType::from_data(vec![31536000])), ("c", DateType::from_data(vec![-50, 0, 50])), - ("d", TimestampType::from_data(vec![215360000000])), + ("d", TimestampType::from_data(vec![21536000])), ]); } From 5fc9299f6e615d9471d1767302b975c0800935a3 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 19:57:29 +0200 Subject: [PATCH 12/64] Correction unit tests 6 --- src/query/expression/src/utils/date_helper.rs | 18 +++++++----------- .../functions/tests/it/scalars/datetime.rs | 16 +++++++--------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index bfc41a8ffaa66..b97a08571d939 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -245,10 +245,9 @@ pub trait DateConverter { fn to_timestamp(&self, tz: Tz) -> DateTime; fn convert_timezone( &self, + src_tz: Option, target_tz: Tz, src_timestamp: DateTime, - src_tz: Option, - src_ntz_timestamp: Option, ) -> DateTime; } @@ -280,23 +279,20 @@ where T: AsPrimitive /// - `src_ntz_timestamp`: Source timestamp with unspecified timezone to be converted - Optional fn convert_timezone( &self, + src_tz: Option, target_tz: Tz, src_timestamp: DateTime, - src_tz: Option, - src_ntz_timestamp: Option, ) -> DateTime { let timestamp_to_convert: DateTime; - if let (Some(ntz_timestamp), Some(tz)) = (src_ntz_timestamp, src_tz) { - timestamp_to_convert = tz.from_local_datetime(&ntz_timestamp).unwrap(); + + if let Some(tz) = src_tz { + let naive_dt = src_timestamp.naive_local(); + timestamp_to_convert = tz.from_local_datetime(&naive_dt).unwrap(); } else { timestamp_to_convert = src_timestamp; } - if timestamp_to_convert.timezone() != target_tz { - timestamp_to_convert.with_timezone(&target_tz) - } else { - timestamp_to_convert - } + timestamp_to_convert.with_timezone(&target_tz) } } diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 3826f2a0d6312..f811679b4b361 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -65,18 +65,16 @@ fn test_to_timestamp(file: &mut impl Write) { } fn test_convert_timezone(file: &mut impl Write) { - run_ast(file, "convert_timezone(a,b,c,d)", &[ - ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", TimestampType::from_data(vec![31536000])), - ("c", DateType::from_data(vec![])), - ("d", TimestampType::from_data(vec![])), + run_ast(file, "convert_timezone(a,b,c)", &[ + ("a", TimestampType::from_data(vec![])), + ("b", DateType::from_data(vec![-100, 0, 100])), + ("c", TimestampType::from_data(vec![31536000])), ]); - run_ast(file, "convert_timezone(a,b,c,d)", &[ + run_ast(file, "convert_timezone(a,b,c)", &[ ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", TimestampType::from_data(vec![31536000])), - ("c", DateType::from_data(vec![-50, 0, 50])), - ("d", TimestampType::from_data(vec![21536000])), + ("b", DateType::from_data(vec![-50, 0, 50])), + ("c", TimestampType::from_data(vec![31536000])), ]); } From 856eef40920d197046f4f63ef108cd7b84484263 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Sun, 4 Aug 2024 20:23:11 +0200 Subject: [PATCH 13/64] Correction unit tests 6 --- src/query/functions/tests/it/scalars/datetime.rs | 14 -------------- .../functions/02_0012_function_datetimes_tz.test | 5 ++++- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index f811679b4b361..f4c14228e98c3 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -64,20 +64,6 @@ fn test_to_timestamp(file: &mut impl Write) { )]); } -fn test_convert_timezone(file: &mut impl Write) { - run_ast(file, "convert_timezone(a,b,c)", &[ - ("a", TimestampType::from_data(vec![])), - ("b", DateType::from_data(vec![-100, 0, 100])), - ("c", TimestampType::from_data(vec![31536000])), - ]); - - run_ast(file, "convert_timezone(a,b,c)", &[ - ("a", DateType::from_data(vec![-100, 0, 100])), - ("b", DateType::from_data(vec![-50, 0, 50])), - ("c", TimestampType::from_data(vec![31536000])), - ]); -} - fn test_to_date(file: &mut impl Write) { run_ast(file, "to_date(-354286)", &[]); run_ast(file, "to_date(-354285)", &[]); diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index f0e6d27187833..56a762ebc96da 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -658,7 +658,6 @@ select to_datetime('1', '%s') statement error 1006 select to_timestamp('200,2000', '%s,%Y'); - statement ok set timezone='UTC'; @@ -769,3 +768,7 @@ query T SELECT substr(DATE_ADD(month, 1, now())::String, 1,4)=substr(now()::String, 1,4); ---- 1 + +statement error 1006 +select convert_timezone('Asia/Shanghai', 'Pacific/Apia', '1947-04-15 00:00:00'); + From 0f7d8345741b4de8747e7f5b2749817f3e459600 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 18:46:53 +0200 Subject: [PATCH 14/64] Implementation of convert_timezone --- src/query/expression/src/utils/date_helper.rs | 30 ----------- src/query/functions/src/scalars/datetime.rs | 53 +++++++++++++++++++ .../02_0012_function_datetimes_tz.test | 2 + 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index b97a08571d939..175d63e185e97 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -243,12 +243,6 @@ impl TzLUT { pub trait DateConverter { fn to_date(&self, tz: Tz) -> NaiveDate; fn to_timestamp(&self, tz: Tz) -> DateTime; - fn convert_timezone( - &self, - src_tz: Option, - target_tz: Tz, - src_timestamp: DateTime, - ) -> DateTime; } impl DateConverter for T @@ -270,30 +264,6 @@ where T: AsPrimitive } tz.timestamp_opt(secs, nanos as u32).unwrap() } - /// Convert a timestamp to the specify timezone. - /// - /// # Parameters - /// - `target_tz`: Timezone in which the timestamp will be converted - /// - `src_timestamp`: Source timestamp to be converted - /// - `src_tz`: Timezone of the timestamp to be converted - Optional - /// - `src_ntz_timestamp`: Source timestamp with unspecified timezone to be converted - Optional - fn convert_timezone( - &self, - src_tz: Option, - target_tz: Tz, - src_timestamp: DateTime, - ) -> DateTime { - let timestamp_to_convert: DateTime; - - if let Some(tz) = src_tz { - let naive_dt = src_timestamp.naive_local(); - timestamp_to_convert = tz.from_local_datetime(&naive_dt).unwrap(); - } else { - timestamp_to_convert = src_timestamp; - } - - timestamp_to_convert.with_timezone(&target_tz) - } } pub const MICROSECS_PER_DAY: i64 = 86_400_000_000; diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index b8755f33c10e8..93feabf2b314a 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -134,6 +134,59 @@ fn int64_domain_to_timestamp_domain>( }) } +fn register_convert_timezone(registry: &mut FunctionRegistry){ + + registry.register_2_arg::( + "convert_timezone", + |_|{TimestampType}, + eval_convert_timezone, + ); + + /*registry.register_3_arg::( + "convert_timezone", + |_| None, + eval_convert_timezone, + );*/ + + fn eval_convert_timezone( + target_tz: ValueRef, + src_timestamp: ValueRef, + ctx: &mut EvalContext, + ) -> Value { + vectorize_with_builder_2_arg::(|target_tz, src_timestamp, output, ctx|{ + /* Parsing parameters */ + let timestamp; + match string_to_timestamp(src_timestamp, ctx.func_ctx.tz.tz, false){ + Ok(ts) => timestamp = ts, + Err(e) => { + ctx.set_error( + output.len(), + format!("cannot parse to type `TIMESTAMP`. {}", e), + ) + } + } + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ) + } + }; + + /* if let Some(tz) = src_tz { + let naive_dt = src_timestamp.naive_local(); + timestamp = tz.from_local_datetime(&naive_dt).unwrap(); + } else { + timestamp = src_timestamp; + }*/ + + output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + })(target_tz, src_timestamp, ctx) + + } +} fn register_string_to_timestamp(registry: &mut FunctionRegistry) { registry.register_aliases("to_date", &["str_to_date", "date"]); registry.register_aliases("to_year", &["str_to_year", "year"]); diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index 56a762ebc96da..e51fd715086a4 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -772,3 +772,5 @@ SELECT substr(DATE_ADD(month, 1, now())::String, 1,4)=substr(now()::String, 1,4) statement error 1006 select convert_timezone('Asia/Shanghai', 'Pacific/Apia', '1947-04-15 00:00:00'); +statement error 1006 +select convert_timezone('Asia/Shanghai', '1947-04-15 00:00:00'); From 71803ab5b20aebea1a26addfbd3f49f4838e25ae Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 19:36:16 +0200 Subject: [PATCH 15/64] Correction registry and error handling --- src/query/functions/src/scalars/datetime.rs | 37 +++++++++++---------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 93feabf2b314a..197f4360f955c 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -136,18 +136,12 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry){ - registry.register_2_arg::( + registry.register_2_arg::( "convert_timezone", - |_|{TimestampType}, + |_, _| FunctionDomain::MayThrow, eval_convert_timezone, ); - /*registry.register_3_arg::( - "convert_timezone", - |_| None, - eval_convert_timezone, - );*/ - fn eval_convert_timezone( target_tz: ValueRef, src_timestamp: ValueRef, @@ -155,25 +149,34 @@ fn register_convert_timezone(registry: &mut FunctionRegistry){ ) -> Value { vectorize_with_builder_2_arg::(|target_tz, src_timestamp, output, ctx|{ /* Parsing parameters */ + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ) + } + }; + let timestamp; match string_to_timestamp(src_timestamp, ctx.func_ctx.tz.tz, false){ Ok(ts) => timestamp = ts, Err(e) => { + + let naivetime = NaiveDateTime::parse_from_str("2023-08-05 12:34:56", "%Y-%m-%d %H:%M:%S").expect("Failed to parse NaiveDateTime"); + let dummy_tz: Tz = "America/New_York".parse().expect("Failed to parse timezone"); + let dummy_utc: DateTime = Utc.from_utc_datetime(&naivetime); + timestamp: DateTime = dummy_tz.from_local_datetime(&naivetime) + .single() + .expect("Failed to convert NaiveDateTime to DateTime with timezone"); + ctx.set_error( output.len(), format!("cannot parse to type `TIMESTAMP`. {}", e), ) } } - let t_tz: Tz = match target_tz.parse() { - Ok(tz) => tz, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ) - } - }; /* if let Some(tz) = src_tz { let naive_dt = src_timestamp.naive_local(); From 33f170b86fd2a73b78fd5c06ab7d033209ee0f3f Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 19:44:01 +0200 Subject: [PATCH 16/64] Correction eval function --- src/query/functions/src/scalars/datetime.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 197f4360f955c..6708607e6c1e3 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -164,10 +164,9 @@ fn register_convert_timezone(registry: &mut FunctionRegistry){ Ok(ts) => timestamp = ts, Err(e) => { - let naivetime = NaiveDateTime::parse_from_str("2023-08-05 12:34:56", "%Y-%m-%d %H:%M:%S").expect("Failed to parse NaiveDateTime"); - let dummy_tz: Tz = "America/New_York".parse().expect("Failed to parse timezone"); - let dummy_utc: DateTime = Utc.from_utc_datetime(&naivetime); - timestamp: DateTime = dummy_tz.from_local_datetime(&naivetime) + let dummy_naivetime: NaiveDateTime = NaiveDateTime::parse_from_str("1970-01-01 00:00:00", "%Y-%m-%d %H:%M:%S").expect("Failed to parse NaiveDateTime"); + let dummy_utc: DateTime = Utc.from_utc_datetime(&dummy_naivetime); + timestamp: DateTime = ctx.func_ctx.tz.tz.from_local_datetime(&dummy_naivetime) .single() .expect("Failed to convert NaiveDateTime to DateTime with timezone"); From 9947d8f4db66ab842d02a61b3a83c483b0da3691 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 19:47:22 +0200 Subject: [PATCH 17/64] Correction eval function 2 --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 6708607e6c1e3..b7148244f14cd 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -166,7 +166,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry){ let dummy_naivetime: NaiveDateTime = NaiveDateTime::parse_from_str("1970-01-01 00:00:00", "%Y-%m-%d %H:%M:%S").expect("Failed to parse NaiveDateTime"); let dummy_utc: DateTime = Utc.from_utc_datetime(&dummy_naivetime); - timestamp: DateTime = ctx.func_ctx.tz.tz.from_local_datetime(&dummy_naivetime) + timestamp = ctx.func_ctx.tz.tz.from_local_datetime(&dummy_naivetime) .single() .expect("Failed to convert NaiveDateTime to DateTime with timezone"); From 5df3e124230aceef7e8d801b4d0dcc53917eb91e Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 19:53:04 +0200 Subject: [PATCH 18/64] Correction eval function 3 --- src/query/functions/src/scalars/datetime.rs | 82 ++++++++++----------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index b7148244f14cd..76ae75d5bd4a4 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -134,8 +134,7 @@ fn int64_domain_to_timestamp_domain>( }) } -fn register_convert_timezone(registry: &mut FunctionRegistry){ - +fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_2_arg::( "convert_timezone", |_, _| FunctionDomain::MayThrow, @@ -143,50 +142,51 @@ fn register_convert_timezone(registry: &mut FunctionRegistry){ ); fn eval_convert_timezone( - target_tz: ValueRef, - src_timestamp: ValueRef, - ctx: &mut EvalContext, + target_tz: ValueRef, + src_timestamp: ValueRef, + ctx: &mut EvalContext, ) -> Value { - vectorize_with_builder_2_arg::(|target_tz, src_timestamp, output, ctx|{ - /* Parsing parameters */ - let t_tz: Tz = match target_tz.parse() { - Ok(tz) => tz, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ) - } - }; - - let timestamp; - match string_to_timestamp(src_timestamp, ctx.func_ctx.tz.tz, false){ - Ok(ts) => timestamp = ts, - Err(e) => { + vectorize_with_builder_2_arg::( + |target_tz, src_timestamp, output, ctx| { + // Parsing parameters + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ); + } + }; - let dummy_naivetime: NaiveDateTime = NaiveDateTime::parse_from_str("1970-01-01 00:00:00", "%Y-%m-%d %H:%M:%S").expect("Failed to parse NaiveDateTime"); - let dummy_utc: DateTime = Utc.from_utc_datetime(&dummy_naivetime); - timestamp = ctx.func_ctx.tz.tz.from_local_datetime(&dummy_naivetime) - .single() - .expect("Failed to convert NaiveDateTime to DateTime with timezone"); + let timestamp; + match string_to_timestamp(src_timestamp, ctx.func_ctx.tz.tz, false) { + Ok(ts) => timestamp = ts, + Err(e) => { + let dummy_naivetime: NaiveDateTime = NaiveDateTime::parse_from_str( + "1970-01-01 00:00:00", + "%Y-%m-%d %H:%M:%S", + ) + .expect("Failed to parse NaiveDateTime"); + let dummy_utc: DateTime = Utc.from_utc_datetime(&dummy_naivetime); + timestamp = ctx + .func_ctx + .tz + .tz + .from_local_datetime(&dummy_naivetime) + .single() + .expect("Failed to convert NaiveDateTime to DateTime with timezone"); - ctx.set_error( - output.len(), - format!("cannot parse to type `TIMESTAMP`. {}", e), - ) + ctx.set_error( + output.len(), + format!("cannot parse to type `TIMESTAMP`. {}", e), + ) + } } - } - - /* if let Some(tz) = src_tz { - let naive_dt = src_timestamp.naive_local(); - timestamp = tz.from_local_datetime(&naive_dt).unwrap(); - } else { - timestamp = src_timestamp; - }*/ - - output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); - })(target_tz, src_timestamp, ctx) + output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + }, + )(target_tz, src_timestamp, ctx) } } fn register_string_to_timestamp(registry: &mut FunctionRegistry) { From 56d9f60b372959e806b359afb4e2139cd86e9873 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 20:00:06 +0200 Subject: [PATCH 19/64] Correction eval function 4 --- src/query/functions/src/scalars/datetime.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 76ae75d5bd4a4..b5de81dc79065 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -137,7 +137,7 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_2_arg::( "convert_timezone", - |_, _| FunctionDomain::MayThrow, + |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); @@ -168,7 +168,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { "%Y-%m-%d %H:%M:%S", ) .expect("Failed to parse NaiveDateTime"); - let dummy_utc: DateTime = Utc.from_utc_datetime(&dummy_naivetime); timestamp = ctx .func_ctx .tz From b2792337bdfa435f22df21ce13b269ce9fffa154 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 20:22:36 +0200 Subject: [PATCH 20/64] Correction eval function 5 --- src/query/functions/src/scalars/datetime.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index b5de81dc79065..b602cb72692e0 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -138,14 +138,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, - eval_convert_timezone, - ); - - fn eval_convert_timezone( - target_tz: ValueRef, - src_timestamp: ValueRef, - ctx: &mut EvalContext, - ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters @@ -185,8 +177,8 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); }, - )(target_tz, src_timestamp, ctx) - } + ), + ); } fn register_string_to_timestamp(registry: &mut FunctionRegistry) { registry.register_aliases("to_date", &["str_to_date", "date"]); From ae5b74df9b9619ee8b02ef51a5b151474d0bd6af Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 20:37:48 +0200 Subject: [PATCH 21/64] Correction eval function 6 --- src/query/functions/src/scalars/datetime.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index b602cb72692e0..69260045f6e86 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -137,7 +137,15 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_2_arg::( "convert_timezone", - |_, _, _| FunctionDomain::MayThrow, + |_, _, _| FunctionDomain::Full, + eval_convert_timezone, + ); + + fn eval_convert_timezone( + target_tz: ValueRef, + src_timestamp: ValueRef, + ctx: &mut EvalContext, + ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters @@ -177,8 +185,8 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); }, - ), - ); + )(target_tz, src_timestamp, ctx) + } } fn register_string_to_timestamp(registry: &mut FunctionRegistry) { registry.register_aliases("to_date", &["str_to_date", "date"]); From 0ec240e4d503e6130fe93b5d53e997156dea9a23 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 20:57:20 +0200 Subject: [PATCH 22/64] Correction eval function 7 --- src/query/functions/src/scalars/datetime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 69260045f6e86..2b93d475c6571 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -137,7 +137,7 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_2_arg::( "convert_timezone", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); @@ -146,7 +146,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { src_timestamp: ValueRef, ctx: &mut EvalContext, ) -> Value { - vectorize_with_builder_2_arg::( + vectorize_with_builder_2_arg::, ValueRef, TimestampType>( |target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { From 707c4454899b853088f41574f3859cd51fb8febc Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 21:10:41 +0200 Subject: [PATCH 23/64] Correction eval function 8 --- src/query/functions/src/scalars/datetime.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 2b93d475c6571..75c89f2356026 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -25,8 +25,8 @@ use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; use databend_common_exception::ErrorCode; -use databend_common_expression::error_to_null; use databend_common_expression::types::date::clamp_date; +use databend_common_expression::{error_to_null, Scalar}; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::date::string_to_date; use databend_common_expression::types::date::DATE_MAX; @@ -142,10 +142,10 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); fn eval_convert_timezone( - target_tz: ValueRef, - src_timestamp: ValueRef, + target_tz: ValueRef::Scalar(StringType), + src_timestamp: ValueRef::Scalar(StringType), ctx: &mut EvalContext, - ) -> Value { + ) -> ValueRef::Scalar(TimestampType) { vectorize_with_builder_2_arg::, ValueRef, TimestampType>( |target_tz, src_timestamp, output, ctx| { // Parsing parameters From 2f9b60bef2834c3181774d245791f37c4ec58a97 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 22:05:15 +0200 Subject: [PATCH 24/64] Correction eval function 9 --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 75c89f2356026..2b904a70d248e 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -146,7 +146,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { src_timestamp: ValueRef::Scalar(StringType), ctx: &mut EvalContext, ) -> ValueRef::Scalar(TimestampType) { - vectorize_with_builder_2_arg::, ValueRef, TimestampType>( + vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { From e1d78981dc83912e0480b94750e1f684e5ed128b Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 23:08:24 +0200 Subject: [PATCH 25/64] Correction eval function 11 --- src/query/functions/src/scalars/datetime.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 2b904a70d248e..53da5df31f658 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -135,17 +135,17 @@ fn int64_domain_to_timestamp_domain>( } fn register_convert_timezone(registry: &mut FunctionRegistry) { - registry.register_2_arg::( + registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); fn eval_convert_timezone( - target_tz: ValueRef::Scalar(StringType), - src_timestamp: ValueRef::Scalar(StringType), + target_tz: ValueRef, + src_timestamp: ValueRef, ctx: &mut EvalContext, - ) -> ValueRef::Scalar(TimestampType) { + ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters From aa9d108b50d52e614861ff15280fd9c37f27712c Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Mon, 5 Aug 2024 23:17:14 +0200 Subject: [PATCH 26/64] Correction eval function 12 --- src/query/functions/src/scalars/datetime.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 53da5df31f658..840607b7557fa 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -110,6 +110,8 @@ pub fn register(registry: &mut FunctionRegistry) { // [date | timestamp] +/- number register_timestamp_add_sub(registry); + + register_convert_timezone(registry); } /// Check if timestamp is within range, and return the timestamp in micros. From b74ad41ba351383c1cc3286afa616c9e1513e7a9 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 13:53:33 +0200 Subject: [PATCH 27/64] Correction eval function 13 + add of the 3 arguments function version --- src/query/functions/src/scalars/datetime.rs | 95 ++++++++++++++++----- 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 840607b7557fa..679c9c6f1c985 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -111,6 +111,7 @@ pub fn register(registry: &mut FunctionRegistry) { // [date | timestamp] +/- number register_timestamp_add_sub(registry); + // convert_timezone( target_timezone, date, src_timezone) register_convert_timezone(registry); } @@ -137,12 +138,20 @@ fn int64_domain_to_timestamp_domain>( } fn register_convert_timezone(registry: &mut FunctionRegistry) { + /* 2 arguments function [target_timezone, src_timestamp] */ registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); + /* 3 arguments function [target_timezone, src_timestamp, src_timezone] */ + registry.register_passthrough_nullable_3_arg::( + "convert_timezone", + |_, _, _, _| FunctionDomain::MayThrow, + eval_convert_timezone_with_src_timezone, + ); + fn eval_convert_timezone( target_tz: ValueRef, src_timestamp: ValueRef, @@ -161,31 +170,77 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let timestamp; - match string_to_timestamp(src_timestamp, ctx.func_ctx.tz.tz, false) { - Ok(ts) => timestamp = ts, + let result_timestamp = match src_timestamp.parse::>() + { + Ok(_) => src_timestamp.parse::>(), Err(e) => { - let dummy_naivetime: NaiveDateTime = NaiveDateTime::parse_from_str( - "1970-01-01 00:00:00", - "%Y-%m-%d %H:%M:%S", - ) - .expect("Failed to parse NaiveDateTime"); - timestamp = ctx - .func_ctx - .tz - .tz - .from_local_datetime(&dummy_naivetime) - .single() - .expect("Failed to convert NaiveDateTime to DateTime with timezone"); + return ctx.set_error( + output.len(), + format!("Unable to parse src_timestamp : {}", e), + ); + } + }; + let timestamp = result_timestamp.unwrap(); + output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + }, + )(target_tz, src_timestamp, ctx) + } - ctx.set_error( + fn eval_convert_timezone_with_src_timezone( + target_tz: ValueRef, + src_timestamp: ValueRef, + src_tz: ValueRef, + ctx: &mut EvalContext, + ) -> Value { + vectorize_with_builder_2_arg::( + |target_tz, src_timestamp, output, ctx| { + // Parsing parameters + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( output.len(), - format!("cannot parse to type `TIMESTAMP`. {}", e), - ) + format!("cannot parse target `timezone`. {}", e), + ); } - } + }; - output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + let s_tz: Tz = match src_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse src `timezone`. {}", e), + ); + } + }; + + let result_timestamp_naive = match src_timestamp.parse::>() + { + Ok(_) => { + return ctx.set_error( + output.len(), + format!("src_timestamp had a timezone"), + ); + }, + Err(_) => { + match src_timestamp.parse::(){ + Ok(naive_datetime) => { + Ok(naive_datetime) + }, + Err(e) =>{ + return ctx.set_error( + output.len(), + format!("Unable to parse src_timestamp : {}", e), + ); + } + } + } + }; + + let timestamp = result_timestamp_naive.unwrap(); + let timestamp_utc = Utc.from_utc_datetime(×tamp); + output.push(timestamp_utc.with_timezone(&t_tz).timestamp_micros()); }, )(target_tz, src_timestamp, ctx) } From 8f6174128dc14b4ad03b952b80698e167ae3d345 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 14:02:06 +0200 Subject: [PATCH 28/64] Code formatting --- src/query/functions/src/scalars/datetime.rs | 42 +++++++++------------ 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 679c9c6f1c985..6aec857e7bfd6 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -138,14 +138,14 @@ fn int64_domain_to_timestamp_domain>( } fn register_convert_timezone(registry: &mut FunctionRegistry) { - /* 2 arguments function [target_timezone, src_timestamp] */ + // 2 arguments function [target_timezone, src_timestamp] registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); - /* 3 arguments function [target_timezone, src_timestamp, src_timezone] */ + // 3 arguments function [target_timezone, src_timestamp, src_timezone] registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, @@ -170,13 +170,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp = match src_timestamp.parse::>() - { + let result_timestamp = match src_timestamp.parse::>() { Ok(_) => src_timestamp.parse::>(), Err(e) => { return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), + output.len(), + format!("Unable to parse src_timestamp : {}", e), ); } }; @@ -215,27 +214,20 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp_naive = match src_timestamp.parse::>() - { + let result_timestamp_naive = match src_timestamp.parse::>() { Ok(_) => { - return ctx.set_error( - output.len(), - format!("src_timestamp had a timezone"), - ); - }, - Err(_) => { - match src_timestamp.parse::(){ - Ok(naive_datetime) => { - Ok(naive_datetime) - }, - Err(e) =>{ - return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), - ); - } - } + return ctx + .set_error(output.len(), format!("src_timestamp had a timezone")); } + Err(_) => match src_timestamp.parse::() { + Ok(naive_datetime) => Ok(naive_datetime), + Err(e) => { + return ctx.set_error( + output.len(), + format!("Unable to parse src_timestamp : {}", e), + ); + } + }, }; let timestamp = result_timestamp_naive.unwrap(); From c4d493ce125351e539865282ff63ba9ca4170eb6 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 14:16:27 +0200 Subject: [PATCH 29/64] Correction third parameter convert_timezone() --- src/query/functions/src/scalars/datetime.rs | 76 +++++++++------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 6aec857e7bfd6..d2e18fb12f9c8 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -58,6 +58,7 @@ use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::EvalContext; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionProperty; @@ -149,16 +150,8 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, - eval_convert_timezone_with_src_timezone, - ); - - fn eval_convert_timezone( - target_tz: ValueRef, - src_timestamp: ValueRef, - ctx: &mut EvalContext, - ) -> Value { - vectorize_with_builder_2_arg::( - |target_tz, src_timestamp, output, ctx| { + vectorize_with_builder_3_arg::( + |target_tz, src_timestamp, src_tz, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, @@ -170,25 +163,42 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp = match src_timestamp.parse::>() { - Ok(_) => src_timestamp.parse::>(), + let s_tz: Tz = match src_tz.parse() { + Ok(tz) => tz, Err(e) => { return ctx.set_error( output.len(), - format!("Unable to parse src_timestamp : {}", e), + format!("cannot parse src `timezone`. {}", e), ); } }; - let timestamp = result_timestamp.unwrap(); - output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + + let result_timestamp_naive = match src_timestamp.parse::>() { + Ok(_) => { + return ctx + .set_error(output.len(), format!("src_timestamp had a timezone")); + } + Err(_) => match src_timestamp.parse::() { + Ok(naive_datetime) => Ok(naive_datetime), + Err(e) => { + return ctx.set_error( + output.len(), + format!("Unable to parse src_timestamp : {}", e), + ); + } + }, + }; + + let timestamp = result_timestamp_naive.unwrap(); + let timestamp_utc = Utc.from_utc_datetime(×tamp); + output.push(timestamp_utc.with_timezone(&t_tz).timestamp_micros()); }, - )(target_tz, src_timestamp, ctx) - } + ), + ); - fn eval_convert_timezone_with_src_timezone( + fn eval_convert_timezone( target_tz: ValueRef, src_timestamp: ValueRef, - src_tz: ValueRef, ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_2_arg::( @@ -204,35 +214,17 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let s_tz: Tz = match src_tz.parse() { - Ok(tz) => tz, + let result_timestamp = match src_timestamp.parse::>() { + Ok(_) => src_timestamp.parse::>(), Err(e) => { return ctx.set_error( output.len(), - format!("cannot parse src `timezone`. {}", e), + format!("Unable to parse src_timestamp : {}", e), ); } }; - - let result_timestamp_naive = match src_timestamp.parse::>() { - Ok(_) => { - return ctx - .set_error(output.len(), format!("src_timestamp had a timezone")); - } - Err(_) => match src_timestamp.parse::() { - Ok(naive_datetime) => Ok(naive_datetime), - Err(e) => { - return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), - ); - } - }, - }; - - let timestamp = result_timestamp_naive.unwrap(); - let timestamp_utc = Utc.from_utc_datetime(×tamp); - output.push(timestamp_utc.with_timezone(&t_tz).timestamp_micros()); + let timestamp = result_timestamp.unwrap(); + output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); }, )(target_tz, src_timestamp, ctx) } From 3edf227c5d6c99d5b45d66af12764314268a2eea Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 14:34:25 +0200 Subject: [PATCH 30/64] Correction type --- src/query/functions/src/scalars/datetime.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index d2e18fb12f9c8..580c46d6ed7ad 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -24,6 +24,7 @@ use chrono::MappedLocalTime; use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; +use databend_common_ast::ParseError; use databend_common_exception::ErrorCode; use databend_common_expression::types::date::clamp_date; use databend_common_expression::{error_to_null, Scalar}; @@ -173,7 +174,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp_naive = match src_timestamp.parse::>() { + let result_timestamp_naive :Result = match src_timestamp.parse::>() { Ok(_) => { return ctx .set_error(output.len(), format!("src_timestamp had a timezone")); From 9b82482450c9f14f9dce3ec7981d572dfdcb81bd Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 14:43:27 +0200 Subject: [PATCH 31/64] Correction 1 --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 580c46d6ed7ad..078a26381f0bd 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -191,7 +191,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { }; let timestamp = result_timestamp_naive.unwrap(); - let timestamp_utc = Utc.from_utc_datetime(×tamp); + let timestamp_utc = Utc.from_utc_datetime(×tamp).with_timezone(&s_tz); output.push(timestamp_utc.with_timezone(&t_tz).timestamp_micros()); }, ), From aeac109a94d1f782521a183e2223f1b578782604 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 15:17:08 +0200 Subject: [PATCH 32/64] Correction 2 --- src/query/functions/src/scalars/datetime.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 078a26381f0bd..0aa4f66e200c1 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -24,7 +24,6 @@ use chrono::MappedLocalTime; use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; -use databend_common_ast::ParseError; use databend_common_exception::ErrorCode; use databend_common_expression::types::date::clamp_date; use databend_common_expression::{error_to_null, Scalar}; @@ -174,7 +173,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp_naive :Result = match src_timestamp.parse::>() { + let result_timestamp_naive :Result = match src_timestamp.parse::>() { Ok(_) => { return ctx .set_error(output.len(), format!("src_timestamp had a timezone")); From 58bb68c9e6e1eea8a595983b44e177ef8c264234 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 15:59:49 +0200 Subject: [PATCH 33/64] Adding unit test --- src/query/functions/tests/it/scalars/datetime.rs | 5 +++++ .../query/functions/02_0012_function_datetimes_tz.test | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index f4c14228e98c3..0c598fd602401 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -39,6 +39,11 @@ fn test_datetime() { test_convert_timezone(file); } +fn test_convert_timezone(file: &mut impl Write) +{ + run_ast(file, "convert_timezone('2024-08-06T14:30:00+02:00','America/New_York')", &[]); +} + fn test_to_timestamp(file: &mut impl Write) { run_ast(file, "to_timestamp(-30610224000000001)", &[]); run_ast(file, "to_timestamp(-315360000000000)", &[]); diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index e51fd715086a4..62b2de34039f6 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -774,3 +774,7 @@ select convert_timezone('Asia/Shanghai', 'Pacific/Apia', '1947-04-15 00:00:00'); statement error 1006 select convert_timezone('Asia/Shanghai', '1947-04-15 00:00:00'); + + +select to_timestamp('200,2000', '%s,%Y'); + From 220c56d2ccdc4de9793a3671dfe3d38a3e309f51 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 16:24:43 +0200 Subject: [PATCH 34/64] Adding unit test 2 --- src/query/functions/tests/it/scalars/datetime.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 0c598fd602401..859ca38ba275f 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -39,9 +39,12 @@ fn test_datetime() { test_convert_timezone(file); } -fn test_convert_timezone(file: &mut impl Write) -{ - run_ast(file, "convert_timezone('2024-08-06T14:30:00+02:00','America/New_York')", &[]); +fn test_convert_timezone(file: &mut impl Write) { + run_ast( + file, + "convert_timezone('2024-08-06T14:30:00+02:00','America/New_York')", + &[], + ); } fn test_to_timestamp(file: &mut impl Write) { From 46121cbe361ceadf634298d12b3f15994bffb425 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 19:13:54 +0200 Subject: [PATCH 35/64] Correciton unit test 4 --- src/query/functions/src/scalars/datetime.rs | 29 +++++++++------------ 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0aa4f66e200c1..28d651a2b5df4 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -173,25 +173,20 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - let result_timestamp_naive :Result = match src_timestamp.parse::>() { - Ok(_) => { - return ctx - .set_error(output.len(), format!("src_timestamp had a timezone")); - } - Err(_) => match src_timestamp.parse::() { - Ok(naive_datetime) => Ok(naive_datetime), - Err(e) => { - return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), - ); - } - }, + // Parsing src_timestamp + let result_timestamp = match src_timestamp.parse::>() { + Ok(utc_dt) => Ok(utc_dt), + Err(_) => src_timestamp.parse::().map(|naive_dt| { + Utc.from_utc_datetime(&naive_dt) + }), }; - let timestamp = result_timestamp_naive.unwrap(); - let timestamp_utc = Utc.from_utc_datetime(×tamp).with_timezone(&s_tz); - output.push(timestamp_utc.with_timezone(&t_tz).timestamp_micros()); + match result_timestamp { + Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).timestamp_micros()), + Err(e) => { + return ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); + } + }; }, ), ); From 875fc240cc02800246f8937d4943c34d68cb425d Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 19:27:47 +0200 Subject: [PATCH 36/64] Correciton unit test 5 --- src/query/functions/src/scalars/datetime.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 28d651a2b5df4..9873f4304f5b7 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -175,12 +175,13 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { // Parsing src_timestamp let result_timestamp = match src_timestamp.parse::>() { - Ok(utc_dt) => Ok(utc_dt), + Ok(utc_dt) => Ok(utc_dt.with_timezone(&s_tz)), Err(_) => src_timestamp.parse::().map(|naive_dt| { - Utc.from_utc_datetime(&naive_dt) + s_tz.from_local_datetime(&naive_dt).single().unwrap() }), }; + match result_timestamp { Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).timestamp_micros()), Err(e) => { From b1a55f2517a5f8d22553f16338260c9c6e0bd916 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 20:18:10 +0200 Subject: [PATCH 37/64] Correciton unit test 6 --- src/query/functions/src/scalars/datetime.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 9873f4304f5b7..7a485fa4f630c 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -181,7 +181,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { }), }; - match result_timestamp { Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).timestamp_micros()), Err(e) => { From 467f1395243fc9d864a8b8df4e554cdb1aad3af2 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 20:27:42 +0200 Subject: [PATCH 38/64] Correciton unit test 7 --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 7a485fa4f630c..e5cb584051dfc 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -184,7 +184,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { match result_timestamp { Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).timestamp_micros()), Err(e) => { - return ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); + ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); } }; }, From b17ba0e47ab383315fb62388098287979935bc61 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Tue, 6 Aug 2024 22:22:22 +0200 Subject: [PATCH 39/64] Correction unit test 8 --- .../functions/tests/it/scalars/testdata/datetime.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 673c15f75c747..c26f13a43025b 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -3358,6 +3358,7 @@ output type : Timestamp output domain : {1630812366000000..=1630812366000000} output : '2021-09-05 03:26:06.000000' +<<<<<<< HEAD ast : date_diff(year, to_date(0), to_date(10000)) raw expr : diff_years(to_date(10000), to_date(0)) @@ -3575,3 +3576,12 @@ output domain : {-31622400..=-31622400} output : -31622400 +======= +ast : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') +raw expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') +checked expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') +optimized expr : '2024-08-06T08:30:00.000000' +output type : Timestamp +output domain : {'2024-08-06T08:30:00.000000'..='2024-08-06T08:30:00.000000'} +output : '2024-08-06 08:30:00.000000' +>>>>>>> d5555c7234 (Correction unit test 8) From 73dc4a67eb27b695c5eb605fe7e13a552eff9e5f Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 09:19:42 +0200 Subject: [PATCH 40/64] Correction checked expr --- src/query/functions/tests/it/scalars/testdata/datetime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index c26f13a43025b..059638f8d3c29 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -3579,7 +3579,7 @@ output : -31622400 ======= ast : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') raw expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') -checked expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') +checked expr : convert_timezone ('2024-08-06T14:30:00+02:00','America/New_York') optimized expr : '2024-08-06T08:30:00.000000' output type : Timestamp output domain : {'2024-08-06T08:30:00.000000'..='2024-08-06T08:30:00.000000'} From 27bc461d360813a46e5142480a2f2ae86048141f Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 15:46:14 +0200 Subject: [PATCH 41/64] Correction convert_timezone --- src/query/functions/src/scalars/datetime.rs | 66 ++++++++++++------- .../tests/it/scalars/testdata/datetime.txt | 5 ++ 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index e5cb584051dfc..3e51596b7dddb 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -140,17 +140,17 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { // 2 arguments function [target_timezone, src_timestamp] - registry.register_passthrough_nullable_2_arg::( + registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); // 3 arguments function [target_timezone, src_timestamp, src_timezone] - registry.register_passthrough_nullable_3_arg::( + registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, - vectorize_with_builder_3_arg::( + vectorize_with_builder_3_arg::( |target_tz, src_timestamp, src_tz, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { @@ -172,17 +172,14 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); } }; - // Parsing src_timestamp let result_timestamp = match src_timestamp.parse::>() { Ok(utc_dt) => Ok(utc_dt.with_timezone(&s_tz)), - Err(_) => src_timestamp.parse::().map(|naive_dt| { - s_tz.from_local_datetime(&naive_dt).single().unwrap() - }), + Err(_) => Ok(src_timestamp.parse::().unwrap().and_local_timezone(s_tz).unwrap()), }; match result_timestamp { - Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).timestamp_micros()), + Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).to_string()), Err(e) => { ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); } @@ -195,31 +192,56 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { target_tz: ValueRef, src_timestamp: ValueRef, ctx: &mut EvalContext, - ) -> Value { - vectorize_with_builder_2_arg::( + ) -> ValueRef { + vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); + return ctx.set_error(output.len(),format!("cannot parse target `timezone`. {}", e)) } }; - let result_timestamp = match src_timestamp.parse::>() { - Ok(_) => src_timestamp.parse::>(), + /* Split the src_timestamp, isolate this timezone if given */ + let str_datetime_part: Vec<&str> = src_timestamp.split(' ').collect(); + if str_datetime_part.len() != 3 + { + return ctx.set_error(output.len(),format!("Invalid src_timestamp format : {}", src_timestamp.to_string())) + } + + let (datetime_part, offset_part) = src_timestamp.rsplit_once(' ').expect("Invalid format"); + + let str_naive_datetime = datetime_part; + let mut naive_datetime = match NaiveDateTime::parse_from_str(str_naive_datetime,"%Y-%m-%d %H:%M:%S"){ + Ok(parsed) => parsed, Err(e) => { - return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), - ); + return ctx.set_error(output.len(),format!("Unable to parse src_timestamp : {}", e)) } }; - let timestamp = result_timestamp.unwrap(); - output.push(timestamp.with_timezone(&t_tz).timestamp_micros()); + + let str_offset = offset_part; + // Process the offset + let offset_sign = &str_offset[0..1]; // Get the sign of the offset (+ or -) + let offset_time = &str_offset[1..]; // Get the time part of the offset + let offset_parts: Vec<&str> = offset_time.split(':').collect(); + let hours: i64 = offset_parts[0].parse().expect("Invalid number for hours"); + let minutes: i64 = offset_parts[1].parse().expect("Invalid number for minutes"); + + // Convert offset to seconds + let total_seconds = hours * 3600 + minutes * 60; + let sec_offset = match offset_sign { + "+" => Duration::seconds(-total_seconds), + "-" => Duration::seconds(total_seconds), + _ => { + return ctx.set_error(output.len(),format!("Invalid offset sign")); + } + }; + + // Add the offset to the naive datetime + let adjusted_datetime = naive_datetime.checked_add_signed(sec_offset).expect("Overflow when adding offset"); + let utc_adjusted_datetime= Utc.from_utc_datetime(&adjusted_datetime); + output.push(utc_adjusted_datetime.with_timezone(&t_tz).to_string()) }, )(target_tz, src_timestamp, ctx) } diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 059638f8d3c29..162263d93ad1b 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -3358,6 +3358,7 @@ output type : Timestamp output domain : {1630812366000000..=1630812366000000} output : '2021-09-05 03:26:06.000000' +<<<<<<< HEAD <<<<<<< HEAD ast : date_diff(year, to_date(0), to_date(10000)) @@ -3579,6 +3580,10 @@ output : -31622400 ======= ast : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') raw expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') +======= +ast : convert_timezone('2024-08-06 14:30:00 +02:00','America/New_York') +raw expr : convert_timezone('2024-08-06 14:30:00 +02:00','America/New_York') +>>>>>>> 191c57dc87 (Correction convert_timezone) checked expr : convert_timezone ('2024-08-06T14:30:00+02:00','America/New_York') optimized expr : '2024-08-06T08:30:00.000000' output type : Timestamp From ee4ee70f82718992d16af6bb84066acb8521cfed Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 15:52:22 +0200 Subject: [PATCH 42/64] Correction convert_timezone 2 --- src/query/functions/src/scalars/datetime.rs | 45 ++++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 3e51596b7dddb..70ad84097ca7f 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -199,26 +199,39 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, Err(e) => { - return ctx.set_error(output.len(),format!("cannot parse target `timezone`. {}", e)) + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ); } }; - /* Split the src_timestamp, isolate this timezone if given */ + // Split the src_timestamp, isolate this timezone if given let str_datetime_part: Vec<&str> = src_timestamp.split(' ').collect(); - if str_datetime_part.len() != 3 - { - return ctx.set_error(output.len(),format!("Invalid src_timestamp format : {}", src_timestamp.to_string())) + if str_datetime_part.len() != 3 { + return ctx.set_error( + output.len(), + format!( + "Invalid src_timestamp format : {}", + src_timestamp.to_string() + ), + ); } - let (datetime_part, offset_part) = src_timestamp.rsplit_once(' ').expect("Invalid format"); + let (datetime_part, offset_part) = + src_timestamp.rsplit_once(' ').expect("Invalid format"); let str_naive_datetime = datetime_part; - let mut naive_datetime = match NaiveDateTime::parse_from_str(str_naive_datetime,"%Y-%m-%d %H:%M:%S"){ - Ok(parsed) => parsed, - Err(e) => { - return ctx.set_error(output.len(),format!("Unable to parse src_timestamp : {}", e)) - } - }; + let mut naive_datetime = + match NaiveDateTime::parse_from_str(str_naive_datetime, "%Y-%m-%d %H:%M:%S") { + Ok(parsed) => parsed, + Err(e) => { + return ctx.set_error( + output.len(), + format!("Unable to parse src_timestamp : {}", e), + ); + } + }; let str_offset = offset_part; // Process the offset @@ -234,13 +247,15 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { "+" => Duration::seconds(-total_seconds), "-" => Duration::seconds(total_seconds), _ => { - return ctx.set_error(output.len(),format!("Invalid offset sign")); + return ctx.set_error(output.len(), format!("Invalid offset sign")); } }; // Add the offset to the naive datetime - let adjusted_datetime = naive_datetime.checked_add_signed(sec_offset).expect("Overflow when adding offset"); - let utc_adjusted_datetime= Utc.from_utc_datetime(&adjusted_datetime); + let adjusted_datetime = naive_datetime + .checked_add_signed(sec_offset) + .expect("Overflow when adding offset"); + let utc_adjusted_datetime = Utc.from_utc_datetime(&adjusted_datetime); output.push(utc_adjusted_datetime.with_timezone(&t_tz).to_string()) }, )(target_tz, src_timestamp, ctx) From 5d7fef4a9c16d2c6526476ed87fa08b9ad090209 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 16:15:52 +0200 Subject: [PATCH 43/64] Correction convert_timezone 3 --- src/query/functions/src/scalars/datetime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 70ad84097ca7f..81285ef5f1348 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -179,7 +179,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { }; match result_timestamp { - Ok(timestamp) => output.push(timestamp.with_timezone(&t_tz).to_string()), + Ok(timestamp) => output.push(×tamp.with_timezone(&t_tz).to_string()), Err(e) => { ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); } @@ -256,7 +256,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { .checked_add_signed(sec_offset) .expect("Overflow when adding offset"); let utc_adjusted_datetime = Utc.from_utc_datetime(&adjusted_datetime); - output.push(utc_adjusted_datetime.with_timezone(&t_tz).to_string()) + output.push(&utc_adjusted_datetime.with_timezone(&t_tz).to_string()) }, )(target_tz, src_timestamp, ctx) } From 231c89c05da08c3beb7a1e61d3cd32643b45c746 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 23:01:04 +0200 Subject: [PATCH 44/64] Correction convert_timezone 4 --- src/query/functions/src/scalars/datetime.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 81285ef5f1348..0e8b3f24c7bcc 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -178,12 +178,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { Err(_) => Ok(src_timestamp.parse::().unwrap().and_local_timezone(s_tz).unwrap()), }; - match result_timestamp { - Ok(timestamp) => output.push(×tamp.with_timezone(&t_tz).to_string()), - Err(e) => { - ctx.set_error(output.len(), format!("Unable to parse src_timestamp : {}", e)); - } - }; + output.push(result_timestamp.with_timezone(&t_tz).timestamp_micros()) }, ), ); @@ -256,7 +251,11 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { .checked_add_signed(sec_offset) .expect("Overflow when adding offset"); let utc_adjusted_datetime = Utc.from_utc_datetime(&adjusted_datetime); - output.push(&utc_adjusted_datetime.with_timezone(&t_tz).to_string()) + output.push( + utc_adjusted_datetime + .with_timezone(&t_tz) + .timestamp_micros(), + ) }, )(target_tz, src_timestamp, ctx) } From 26c808fdfcfb367cdad16ea571d7b9ec602333ff Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 23:08:30 +0200 Subject: [PATCH 45/64] Correction convert_timezone 5 --- src/query/functions/src/scalars/datetime.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0e8b3f24c7bcc..f534efeb5b62b 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -140,17 +140,17 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { // 2 arguments function [target_timezone, src_timestamp] - registry.register_passthrough_nullable_2_arg::( + registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); // 3 arguments function [target_timezone, src_timestamp, src_timezone] - registry.register_passthrough_nullable_3_arg::( + registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, - vectorize_with_builder_3_arg::( + vectorize_with_builder_3_arg::( |target_tz, src_timestamp, src_tz, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { @@ -187,8 +187,8 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { target_tz: ValueRef, src_timestamp: ValueRef, ctx: &mut EvalContext, - ) -> ValueRef { - vectorize_with_builder_2_arg::( + ) -> Value { + vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { From e8a9ee9e5d5aa3f1e3b3264aa2d67b08a67f6a7f Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 23:17:29 +0200 Subject: [PATCH 46/64] Correction convert_timezone 6 --- src/query/functions/src/scalars/datetime.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index f534efeb5b62b..e32eb5a20902b 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -68,6 +68,7 @@ use databend_common_expression::ValueRef; use databend_common_io::cursor_ext::unwrap_local_time; use dtparse::parse; use num_traits::AsPrimitive; +use databend_common_ast::ParseError; pub fn register(registry: &mut FunctionRegistry) { // cast(xx AS timestamp) @@ -173,12 +174,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; // Parsing src_timestamp - let result_timestamp = match src_timestamp.parse::>() { + let result_timestamp: Result, ParseError> = match src_timestamp.parse::>() { Ok(utc_dt) => Ok(utc_dt.with_timezone(&s_tz)), Err(_) => Ok(src_timestamp.parse::().unwrap().and_local_timezone(s_tz).unwrap()), }; - output.push(result_timestamp.with_timezone(&t_tz).timestamp_micros()) + output.push(result_timestamp.unwrap().with_timezone(&t_tz).timestamp_micros()) }, ), ); From cff049b9dd4806cc89f3b0c5b5dce13624e0178f Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 23:23:23 +0200 Subject: [PATCH 47/64] Correction convert_timezone 7 --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index e32eb5a20902b..2c87cd7d6a8d8 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -24,6 +24,7 @@ use chrono::MappedLocalTime; use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; +use databend_common_ast::ParseError; use databend_common_exception::ErrorCode; use databend_common_expression::types::date::clamp_date; use databend_common_expression::{error_to_null, Scalar}; @@ -68,7 +69,6 @@ use databend_common_expression::ValueRef; use databend_common_io::cursor_ext::unwrap_local_time; use dtparse::parse; use num_traits::AsPrimitive; -use databend_common_ast::ParseError; pub fn register(registry: &mut FunctionRegistry) { // cast(xx AS timestamp) From aadb453fec4fe4afee60aaf56fa1d55f5f15659c Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Wed, 7 Aug 2024 23:34:13 +0200 Subject: [PATCH 48/64] Correction convert_timezone 8 --- src/query/functions/src/scalars/datetime.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 2c87cd7d6a8d8..9461c2870f220 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -207,10 +207,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { if str_datetime_part.len() != 3 { return ctx.set_error( output.len(), - format!( - "Invalid src_timestamp format : {}", - src_timestamp.to_string() - ), + format!("Invalid src_timestamp format : {}", src_timestamp), ); } @@ -218,7 +215,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { src_timestamp.rsplit_once(' ').expect("Invalid format"); let str_naive_datetime = datetime_part; - let mut naive_datetime = + let naive_datetime = match NaiveDateTime::parse_from_str(str_naive_datetime, "%Y-%m-%d %H:%M:%S") { Ok(parsed) => parsed, Err(e) => { From 05de10ed537093fc790784ee3cc5620ae13110fa Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 19:07:11 +0200 Subject: [PATCH 49/64] Correction convert_timezone 9 --- src/query/functions/src/scalars/datetime.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 9461c2870f220..c38a3ed76caa3 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -24,7 +24,6 @@ use chrono::MappedLocalTime; use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; -use databend_common_ast::ParseError; use databend_common_exception::ErrorCode; use databend_common_expression::types::date::clamp_date; use databend_common_expression::{error_to_null, Scalar}; From ab38264ae508b92ded6fe84441029157b55c907e Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 19:30:44 +0200 Subject: [PATCH 50/64] Correction convert_timezone 10 --- src/query/functions/src/scalars/datetime.rs | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index c38a3ed76caa3..48639309d2eb8 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -140,18 +140,18 @@ fn int64_domain_to_timestamp_domain>( fn register_convert_timezone(registry: &mut FunctionRegistry) { // 2 arguments function [target_timezone, src_timestamp] - registry.register_passthrough_nullable_2_arg::( + registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, eval_convert_timezone, ); // 3 arguments function [target_timezone, src_timestamp, src_timezone] - registry.register_passthrough_nullable_3_arg::( + registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, - vectorize_with_builder_3_arg::( - |target_tz, src_timestamp, src_tz, output, ctx| { + vectorize_with_builder_3_arg::( + |src_tz, target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, @@ -173,9 +173,17 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; // Parsing src_timestamp - let result_timestamp: Result, ParseError> = match src_timestamp.parse::>() { + let result_timestamp: Result, _> = match src_timestamp.parse::>() { Ok(utc_dt) => Ok(utc_dt.with_timezone(&s_tz)), - Err(_) => Ok(src_timestamp.parse::().unwrap().and_local_timezone(s_tz).unwrap()), + Err(_) => match src_timestamp.parse::() { + Ok(naive_dt) => naive_dt.and_local_timezone(s_tz).single(), + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse `src_timestamp`. {}", e), + ); + } + }, }; output.push(result_timestamp.unwrap().with_timezone(&t_tz).timestamp_micros()) @@ -185,10 +193,10 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { fn eval_convert_timezone( target_tz: ValueRef, - src_timestamp: ValueRef, + src_timestamp: TimestampType, ctx: &mut EvalContext, ) -> Value { - vectorize_with_builder_2_arg::( + vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Parsing parameters let t_tz: Tz = match target_tz.parse() { From bfe224c3f7243b9e228fff5e88f7c56f22fc7fc6 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 20:10:11 +0200 Subject: [PATCH 51/64] Correction convert_timezone 11 --- src/query/functions/src/scalars/datetime.rs | 87 +++++---------------- 1 file changed, 21 insertions(+), 66 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 48639309d2eb8..c26bdf6582d05 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -162,7 +162,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); } }; - let s_tz: Tz = match src_tz.parse() { Ok(tz) => tz, Err(e) => { @@ -172,28 +171,24 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); } }; - // Parsing src_timestamp - let result_timestamp: Result, _> = match src_timestamp.parse::>() { - Ok(utc_dt) => Ok(utc_dt.with_timezone(&s_tz)), - Err(_) => match src_timestamp.parse::() { - Ok(naive_dt) => naive_dt.and_local_timezone(s_tz).single(), - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse `src_timestamp`. {}", e), - ); - } - }, - }; + // Create dummy timezone + let utc_now: DateTime = Utc::now(); + + let src_time = utc_now.with_timezone(&s_tz); + let target_time = utc_now.with_timezone(&t_tz); + + // Calculate the difference in seconds + let delta = target_time.signed_duration_since(src_time); + let result_timestamp = src_timestamp + delta.num_seconds(); - output.push(result_timestamp.unwrap().with_timezone(&t_tz).timestamp_micros()) + output.push(result_timestamp) }, ), ); fn eval_convert_timezone( target_tz: ValueRef, - src_timestamp: TimestampType, + src_timestamp: ValueRef, ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_2_arg::( @@ -209,58 +204,18 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - // Split the src_timestamp, isolate this timezone if given - let str_datetime_part: Vec<&str> = src_timestamp.split(' ').collect(); - if str_datetime_part.len() != 3 { - return ctx.set_error( - output.len(), - format!("Invalid src_timestamp format : {}", src_timestamp), - ); - } + // Assume the source timestamp is in UTC + let utc_datetime: DateTime = Utc.timestamp(src_timestamp, 0); - let (datetime_part, offset_part) = - src_timestamp.rsplit_once(' ').expect("Invalid format"); + // Parse the target timezone + let target_timezone: Tz = + target_tz.parse().expect("Failed to parse target timezone"); - let str_naive_datetime = datetime_part; - let naive_datetime = - match NaiveDateTime::parse_from_str(str_naive_datetime, "%Y-%m-%d %H:%M:%S") { - Ok(parsed) => parsed, - Err(e) => { - return ctx.set_error( - output.len(), - format!("Unable to parse src_timestamp : {}", e), - ); - } - }; - - let str_offset = offset_part; - // Process the offset - let offset_sign = &str_offset[0..1]; // Get the sign of the offset (+ or -) - let offset_time = &str_offset[1..]; // Get the time part of the offset - let offset_parts: Vec<&str> = offset_time.split(':').collect(); - let hours: i64 = offset_parts[0].parse().expect("Invalid number for hours"); - let minutes: i64 = offset_parts[1].parse().expect("Invalid number for minutes"); - - // Convert offset to seconds - let total_seconds = hours * 3600 + minutes * 60; - let sec_offset = match offset_sign { - "+" => Duration::seconds(-total_seconds), - "-" => Duration::seconds(total_seconds), - _ => { - return ctx.set_error(output.len(), format!("Invalid offset sign")); - } - }; - - // Add the offset to the naive datetime - let adjusted_datetime = naive_datetime - .checked_add_signed(sec_offset) - .expect("Overflow when adding offset"); - let utc_adjusted_datetime = Utc.from_utc_datetime(&adjusted_datetime); - output.push( - utc_adjusted_datetime - .with_timezone(&t_tz) - .timestamp_micros(), - ) + // Convert the UTC time to the specified target timezone + let target_datetime: DateTime = utc_datetime.with_timezone(&target_timezone); + let result_timestamp = target_datetime.timestamp(); + // Return the adjusted timestamp as a Unix timestamp in seconds + output.push(result_timestamp) }, )(target_tz, src_timestamp, ctx) } From 5ce929f7549344131a3f4c6a374722b07b594d9e Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 20:20:50 +0200 Subject: [PATCH 52/64] Correction convert_timezone 12 --- src/query/functions/src/scalars/datetime.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index c26bdf6582d05..9e98ffe8e0f5c 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -193,26 +193,15 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { - // Parsing parameters - let t_tz: Tz = match target_tz.parse() { - Ok(tz) => tz, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); - } - }; - // Assume the source timestamp is in UTC - let utc_datetime: DateTime = Utc.timestamp(src_timestamp, 0); + let utc_datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); // Parse the target timezone - let target_timezone: Tz = + let t_tz: Tz = target_tz.parse().expect("Failed to parse target timezone"); // Convert the UTC time to the specified target timezone - let target_datetime: DateTime = utc_datetime.with_timezone(&target_timezone); + let target_datetime: DateTime = utc_datetime.with_timezone(&t_tz); let result_timestamp = target_datetime.timestamp(); // Return the adjusted timestamp as a Unix timestamp in seconds output.push(result_timestamp) From 66f61ad19c4c95228759ff49b2b8930038a6e57c Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 20:21:06 +0200 Subject: [PATCH 53/64] Correction convert_timezone 12 --- src/query/functions/src/scalars/datetime.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 9e98ffe8e0f5c..e850dd9c91337 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -197,8 +197,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let utc_datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); // Parse the target timezone - let t_tz: Tz = - target_tz.parse().expect("Failed to parse target timezone"); + let t_tz: Tz = target_tz.parse().expect("Failed to parse target timezone"); // Convert the UTC time to the specified target timezone let target_datetime: DateTime = utc_datetime.with_timezone(&t_tz); From e50bb78cf45c659d3297e15d2de872b472e7cf92 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Thu, 8 Aug 2024 20:55:53 +0200 Subject: [PATCH 54/64] Correction convert_timezone 13 --- src/query/functions/src/scalars/datetime.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index e850dd9c91337..44d62edef1bf0 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -197,7 +197,15 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let utc_datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); // Parse the target timezone - let t_tz: Tz = target_tz.parse().expect("Failed to parse target timezone"); + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ); + } + }; // Convert the UTC time to the specified target timezone let target_datetime: DateTime = utc_datetime.with_timezone(&t_tz); From e45aba209208c7bc2b0405e4e009fc206d6a2a53 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 12:38:32 +0200 Subject: [PATCH 55/64] Unit test --- src/query/functions/tests/it/scalars/datetime.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 859ca38ba275f..31b332c80838b 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -42,9 +42,12 @@ fn test_datetime() { fn test_convert_timezone(file: &mut impl Write) { run_ast( file, - "convert_timezone('2024-08-06T14:30:00+02:00','America/New_York')", + "convert_timezone('America/New_York', 315360000000)", &[], ); + run_ast(file, "convert_timezone('America/New_York', -100)", &[]); + run_ast(file, "convert_timezone('America/New_York', 0)", &[]); + run_ast(file, "convert_timezone('America/New_York', 100)", &[]); } fn test_to_timestamp(file: &mut impl Write) { From e34f6c137e50ac2a1197d88daa24396f87211bcd Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 13:05:19 +0200 Subject: [PATCH 56/64] Unit test 2 --- src/query/functions/tests/it/scalars/datetime.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 31b332c80838b..cb1cc9a2a4985 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -48,6 +48,22 @@ fn test_convert_timezone(file: &mut impl Write) { run_ast(file, "convert_timezone('America/New_York', -100)", &[]); run_ast(file, "convert_timezone('America/New_York', 0)", &[]); run_ast(file, "convert_timezone('America/New_York', 100)", &[]); + + run_ast( + file, + "convert_timezone('America/New_York', 'Europe/Simferopol', 315360000000)", + &[], + ); + run_ast( + file, + "convert_timezone('America/New_York', 'Europe/Simferopol', 100)", + &[], + ); + run_ast( + file, + "convert_timezone('America/New_York', 'Europe/Simferopol', 0)", + &[], + ); } fn test_to_timestamp(file: &mut impl Write) { From fd69ab3e1d8c26c02be5a8ed06ac6d0cb354fff3 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 13:49:44 +0200 Subject: [PATCH 57/64] Unit test 3 --- .../functions/tests/it/scalars/datetime.rs | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index cb1cc9a2a4985..a5dfbc53d9108 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -42,26 +42,34 @@ fn test_datetime() { fn test_convert_timezone(file: &mut impl Write) { run_ast( file, - "convert_timezone('America/New_York', 315360000000)", + "convert_timezone('America/New_York', to_timestamp(100))", + &[], + ); + run_ast( + file, + "convert_timezone('America/New_York', to_timestamp(-100))", &[], ); - run_ast(file, "convert_timezone('America/New_York', -100)", &[]); - run_ast(file, "convert_timezone('America/New_York', 0)", &[]); - run_ast(file, "convert_timezone('America/New_York', 100)", &[]); - run_ast( file, - "convert_timezone('America/New_York', 'Europe/Simferopol', 315360000000)", + "convert_timezone('America/New_York', to_timestamp(0))", &[], ); run_ast( file, - "convert_timezone('America/New_York', 'Europe/Simferopol', 100)", + "convert_timezone('America/New_York', to_timestamp(315360000000))", &[], ); + + run_ast( + file, + "convert_timezone('America/New_York', 'Europe/Simferopol', to_timestamp(100))", + &[], + ); + run_ast( file, - "convert_timezone('America/New_York', 'Europe/Simferopol', 0)", + "convert_timezone('America/New_York', 'Europe/Simferopol', to_timestamp(0))", &[], ); } From 4eaa1103eb69e174e00cea0b07bd6614651bc6f2 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 16:32:31 +0200 Subject: [PATCH 58/64] Unit test 4 --- src/query/functions/src/scalars/datetime.rs | 30 ++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 44d62edef1bf0..8a8956439ba03 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -146,7 +146,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { eval_convert_timezone, ); - // 3 arguments function [target_timezone, src_timestamp, src_timezone] + // 3 arguments function [src_timezone, target_timezone, src_timestamp] registry.register_passthrough_nullable_3_arg::( "convert_timezone", |_, _, _, _| FunctionDomain::MayThrow, @@ -171,6 +171,20 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); } }; + + let p_src_timestamp: i64 = match Some(src_timestamp){ + Ok(timestamp) => { + timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); + }, + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ); + } + }; + + // Create dummy timezone let utc_now: DateTime = Utc::now(); @@ -179,7 +193,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { // Calculate the difference in seconds let delta = target_time.signed_duration_since(src_time); - let result_timestamp = src_timestamp + delta.num_seconds(); + let result_timestamp = p_src_timestamp + delta.num_seconds(); output.push(result_timestamp) }, @@ -194,7 +208,17 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { // Assume the source timestamp is in UTC - let utc_datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); + let utc_datetime: DateTime = match Some(src_timestamp) { + Ok(timestamp) => { + timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); + } + Err(e) => { + return ctx.set_error( + output.len(), + format!("cannot parse target `timezone`. {}", e), + ); + } + }; // Parse the target timezone let t_tz: Tz = match target_tz.parse() { From 0e38b10866b35bf6c2f013f51569f5bae740ab2a Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 16:46:09 +0200 Subject: [PATCH 59/64] Unit test 5 --- src/query/functions/src/scalars/datetime.rs | 41 +++++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 8a8956439ba03..52eebb4b03ecf 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -155,6 +155,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { // Parsing parameters let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, + None => { + return ctx.set_error( + output.len(), + "`target_tz` is `None`.".to_string(), + ); + } Err(e) => { return ctx.set_error( output.len(), @@ -164,6 +170,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { }; let s_tz: Tz = match src_tz.parse() { Ok(tz) => tz, + None => { + return ctx.set_error( + output.len(), + "`src_tz` is `None`.".to_string(), + ); + } Err(e) => { return ctx.set_error( output.len(), @@ -174,8 +186,14 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let p_src_timestamp: i64 = match Some(src_timestamp){ Ok(timestamp) => { - timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); + timestamp.unwrap().Utc.timestamp_opt(src_timestamp, 0).unwrap(); }, + None => { + return ctx.set_error( + output.len(), + "source `src_timestamp` is `None`.".to_string(), + ); + } Err(e) => { return ctx.set_error( output.len(), @@ -209,13 +227,25 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { |target_tz, src_timestamp, output, ctx| { // Assume the source timestamp is in UTC let utc_datetime: DateTime = match Some(src_timestamp) { - Ok(timestamp) => { - timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); + Ok(timestamp) => match Utc.timestamp_opt(timestamp, 0) { + chrono::LocalResult::Single(dt) => dt, + _ => { + return ctx.set_error( + output.len(), + "cannot parse source `src_timestamp`.".to_string(), + ); + } + }, + None => { + return ctx.set_error( + output.len(), + "source `src_timestamp` is `None`.".to_string(), + ); } Err(e) => { return ctx.set_error( output.len(), - format!("cannot parse target `timezone`. {}", e), + format!("cannot parse target `src_timestamp`. {}", e), ); } }; @@ -223,6 +253,9 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { // Parse the target timezone let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, + None => { + return ctx.set_error(output.len(), "`target_tz` is `None`.".to_string()); + } Err(e) => { return ctx.set_error( output.len(), From 86495ea007561a473a5717c21bcc2f02e4457a78 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 16:57:26 +0200 Subject: [PATCH 60/64] Unit test 6 --- src/query/functions/src/scalars/datetime.rs | 27 --------------------- 1 file changed, 27 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 52eebb4b03ecf..3c98e5b0bbe8e 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -161,21 +161,9 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { "`target_tz` is `None`.".to_string(), ); } - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); - } }; let s_tz: Tz = match src_tz.parse() { Ok(tz) => tz, - None => { - return ctx.set_error( - output.len(), - "`src_tz` is `None`.".to_string(), - ); - } Err(e) => { return ctx.set_error( output.len(), @@ -188,12 +176,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { Ok(timestamp) => { timestamp.unwrap().Utc.timestamp_opt(src_timestamp, 0).unwrap(); }, - None => { - return ctx.set_error( - output.len(), - "source `src_timestamp` is `None`.".to_string(), - ); - } Err(e) => { return ctx.set_error( output.len(), @@ -242,20 +224,11 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { "source `src_timestamp` is `None`.".to_string(), ); } - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `src_timestamp`. {}", e), - ); - } }; // Parse the target timezone let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, - None => { - return ctx.set_error(output.len(), "`target_tz` is `None`.".to_string()); - } Err(e) => { return ctx.set_error( output.len(), From 12fb5fa9910abaf7c85da2be39bc0ceb581c1558 Mon Sep 17 00:00:00 2001 From: Reclaimers Date: Fri, 9 Aug 2024 17:18:10 +0200 Subject: [PATCH 61/64] Unit test 7 --- src/query/functions/src/scalars/datetime.rs | 45 +++++++++------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 3c98e5b0bbe8e..0ece099bdce9a 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -155,10 +155,10 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { // Parsing parameters let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, - None => { + Err(e) => { return ctx.set_error( output.len(), - "`target_tz` is `None`.".to_string(), + format!("cannot parse target `timezone`. {}", e), ); } }; @@ -174,7 +174,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let p_src_timestamp: i64 = match Some(src_timestamp){ Ok(timestamp) => { - timestamp.unwrap().Utc.timestamp_opt(src_timestamp, 0).unwrap(); + timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); }, Err(e) => { return ctx.set_error( @@ -207,28 +207,29 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { - // Assume the source timestamp is in UTC - let utc_datetime: DateTime = match Some(src_timestamp) { - Ok(timestamp) => match Utc.timestamp_opt(timestamp, 0) { - chrono::LocalResult::Single(dt) => dt, - _ => { - return ctx.set_error( - output.len(), - "cannot parse source `src_timestamp`.".to_string(), - ); - } - }, - None => { + // Parse the target timezone + let t_tz: Tz = match target_tz.parse() { + Ok(tz) => tz, + Err(e) => { return ctx.set_error( output.len(), - "source `src_timestamp` is `None`.".to_string(), + format!("cannot parse target `timezone`. {}", e), ); } }; - // Parse the target timezone - let t_tz: Tz = match target_tz.parse() { - Ok(tz) => tz, + // Assume the source timestamp is in UTC + match Some(src_timestamp) { + Ok(src_timestamp) => { + let timestamp: i64 = src_timestamp; + let datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); + + // Convert the UTC time to the specified target timezone + let target_datetime: DateTime = datetime.with_timezone(&t_tz); + let result_timestamp = target_datetime.timestamp(); + // Return the adjusted timestamp as a Unix timestamp in seconds + output.push(result_timestamp) + } Err(e) => { return ctx.set_error( output.len(), @@ -236,12 +237,6 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { ); } }; - - // Convert the UTC time to the specified target timezone - let target_datetime: DateTime = utc_datetime.with_timezone(&t_tz); - let result_timestamp = target_datetime.timestamp(); - // Return the adjusted timestamp as a Unix timestamp in seconds - output.push(result_timestamp) }, )(target_tz, src_timestamp, ctx) } From 1cee2cf0c30e4954edac715d6e4d59d7b7b98b86 Mon Sep 17 00:00:00 2001 From: TCeason Date: Fri, 1 Nov 2024 11:52:46 +0800 Subject: [PATCH 62/64] feat: convert_timezone(target_timezone, ts) --- src/query/functions/src/scalars/datetime.rs | 103 +++--------------- .../functions/tests/it/scalars/datetime.rs | 36 ------ .../tests/it/scalars/testdata/datetime.txt | 15 --- .../02_0012_function_datetimes_tz.test | 48 +++++++- 4 files changed, 62 insertions(+), 140 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0ece099bdce9a..d7b8f1478c226 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -25,8 +25,8 @@ use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE; use databend_common_exception::ErrorCode; +use databend_common_expression::error_to_null; use databend_common_expression::types::date::clamp_date; -use databend_common_expression::{error_to_null, Scalar}; use databend_common_expression::types::date::date_to_string; use databend_common_expression::types::date::string_to_date; use databend_common_expression::types::date::DATE_MAX; @@ -58,7 +58,6 @@ use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; -use databend_common_expression::vectorize_with_builder_3_arg; use databend_common_expression::EvalContext; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionProperty; @@ -143,71 +142,11 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_2_arg::( "convert_timezone", |_, _, _| FunctionDomain::MayThrow, - eval_convert_timezone, - ); - - // 3 arguments function [src_timezone, target_timezone, src_timestamp] - registry.register_passthrough_nullable_3_arg::( - "convert_timezone", - |_, _, _, _| FunctionDomain::MayThrow, - vectorize_with_builder_3_arg::( - |src_tz, target_tz, src_timestamp, output, ctx| { - // Parsing parameters - let t_tz: Tz = match target_tz.parse() { - Ok(tz) => tz, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); - } - }; - let s_tz: Tz = match src_tz.parse() { - Ok(tz) => tz, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse src `timezone`. {}", e), - ); - } - }; - - let p_src_timestamp: i64 = match Some(src_timestamp){ - Ok(timestamp) => { - timestamp.Utc.timestamp_opt(src_timestamp, 0).unwrap(); - }, - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); - } - }; - - - // Create dummy timezone - let utc_now: DateTime = Utc::now(); - - let src_time = utc_now.with_timezone(&s_tz); - let target_time = utc_now.with_timezone(&t_tz); - - // Calculate the difference in seconds - let delta = target_time.signed_duration_since(src_time); - let result_timestamp = p_src_timestamp + delta.num_seconds(); - - output.push(result_timestamp) - }, - ), - ); - - fn eval_convert_timezone( - target_tz: ValueRef, - src_timestamp: ValueRef, - ctx: &mut EvalContext, - ) -> Value { vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { - // Parse the target timezone + // Convert source timestamp from source timezone to target timezone + let p_src_timestamp = src_timestamp.to_timestamp(ctx.func_ctx.tz.tz); + let src_dst_from_utc = p_src_timestamp.offset().fix().local_minus_utc(); let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, Err(e) => { @@ -218,29 +157,21 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } }; - // Assume the source timestamp is in UTC - match Some(src_timestamp) { - Ok(src_timestamp) => { - let timestamp: i64 = src_timestamp; - let datetime: DateTime = Utc.timestamp_opt(src_timestamp, 0).unwrap(); - - // Convert the UTC time to the specified target timezone - let target_datetime: DateTime = datetime.with_timezone(&t_tz); - let result_timestamp = target_datetime.timestamp(); - // Return the adjusted timestamp as a Unix timestamp in seconds - output.push(result_timestamp) - } - Err(e) => { - return ctx.set_error( - output.len(), - format!("cannot parse target `timezone`. {}", e), - ); - } - }; + let result_timestamp = p_src_timestamp.with_timezone(&t_tz).timestamp_micros(); + let target_dst_from_utc = p_src_timestamp + .with_timezone(&t_tz) + .offset() + .fix() + .local_minus_utc(); + let offset_as_micros_sec = + (target_dst_from_utc - src_dst_from_utc) as i64 * MICROS_PER_SEC; + let res = result_timestamp.checked_add(offset_as_micros_sec).unwrap(); + output.push(res) }, - )(target_tz, src_timestamp, ctx) - } + ), + ); } + fn register_string_to_timestamp(registry: &mut FunctionRegistry) { registry.register_aliases("to_date", &["str_to_date", "date"]); registry.register_aliases("to_year", &["str_to_year", "year"]); diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index a5dfbc53d9108..783dc6b1f0ef8 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -36,42 +36,6 @@ fn test_datetime() { test_to_number(file); test_rounder_functions(file); test_date_date_diff(file); - test_convert_timezone(file); -} - -fn test_convert_timezone(file: &mut impl Write) { - run_ast( - file, - "convert_timezone('America/New_York', to_timestamp(100))", - &[], - ); - run_ast( - file, - "convert_timezone('America/New_York', to_timestamp(-100))", - &[], - ); - run_ast( - file, - "convert_timezone('America/New_York', to_timestamp(0))", - &[], - ); - run_ast( - file, - "convert_timezone('America/New_York', to_timestamp(315360000000))", - &[], - ); - - run_ast( - file, - "convert_timezone('America/New_York', 'Europe/Simferopol', to_timestamp(100))", - &[], - ); - - run_ast( - file, - "convert_timezone('America/New_York', 'Europe/Simferopol', to_timestamp(0))", - &[], - ); } fn test_to_timestamp(file: &mut impl Write) { diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 162263d93ad1b..673c15f75c747 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -3358,8 +3358,6 @@ output type : Timestamp output domain : {1630812366000000..=1630812366000000} output : '2021-09-05 03:26:06.000000' -<<<<<<< HEAD -<<<<<<< HEAD ast : date_diff(year, to_date(0), to_date(10000)) raw expr : diff_years(to_date(10000), to_date(0)) @@ -3577,16 +3575,3 @@ output domain : {-31622400..=-31622400} output : -31622400 -======= -ast : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') -raw expr : convert_timezone('2024-08-06T14:30:00+02:00','America/New_York') -======= -ast : convert_timezone('2024-08-06 14:30:00 +02:00','America/New_York') -raw expr : convert_timezone('2024-08-06 14:30:00 +02:00','America/New_York') ->>>>>>> 191c57dc87 (Correction convert_timezone) -checked expr : convert_timezone ('2024-08-06T14:30:00+02:00','America/New_York') -optimized expr : '2024-08-06T08:30:00.000000' -output type : Timestamp -output domain : {'2024-08-06T08:30:00.000000'..='2024-08-06T08:30:00.000000'} -output : '2024-08-06 08:30:00.000000' ->>>>>>> d5555c7234 (Correction unit test 8) diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index 62b2de34039f6..f31d1fce57265 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -769,12 +769,54 @@ SELECT substr(DATE_ADD(month, 1, now())::String, 1,4)=substr(now()::String, 1,4) ---- 1 -statement error 1006 -select convert_timezone('Asia/Shanghai', 'Pacific/Apia', '1947-04-15 00:00:00'); + +statement ok +set timezone='Asia/Shanghai'; + +query T +SELECT convert_timezone('America/Los_Angeles', '2024-11-01 11:36:10') +---- +2024-10-31 20:36:10.000000 + +statement ok +set timezone='UTC'; + +statement ok +create or replace table t(c timestamp); + +statement ok +insert into t values('1970-01-01 00:00:00'), ('2024-10-31 22:21:15'); + +# UTC to America/Los_Angeles, 20240-03-10 ~ 2024-11-03 is UTC-7(dst), other is UTC-8 +query T +select c, CONVERT_TIMEZONE('America/Los_Angeles', c) from t; +---- +1970-01-01 00:00:00.000000 1969-12-31 16:00:00.000000 +2024-10-31 22:21:15.000000 2024-10-31 15:21:15.000000 + + +statement ok +set timezone='Asia/Shanghai'; + statement error 1006 select convert_timezone('Asia/Shanghai', '1947-04-15 00:00:00'); +statement ok +set enable_dst_hour_fix=1; -select to_timestamp('200,2000', '%s,%Y'); +# 1947-04-15 00:00:00 is not exists in Asia/Shanghai. Such timings cannot be guaranteed to meet completely +# consider use date_add/sub calc the offset. +query T +select convert_timezone('UTC', '1947-04-15 00:00:00'); +---- +1947-04-14 15:00:00.000000 +statement ok +unset enable_dst_hour_fix; + +statement ok +drop table if exists t; + +statement ok +unset timezone; From 658141d108b1f2a510ad68e28459e1bb5313c405 Mon Sep 17 00:00:00 2001 From: TCeason Date: Fri, 1 Nov 2024 15:11:07 +0800 Subject: [PATCH 63/64] add some check --- src/query/functions/src/scalars/datetime.rs | 20 ++++++++++++++----- .../it/scalars/testdata/function_list.txt | 2 ++ .../02_0012_function_datetimes_tz.test | 16 +++++++++------ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index d7b8f1478c226..2b595d8021d13 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -111,7 +111,7 @@ pub fn register(registry: &mut FunctionRegistry) { // [date | timestamp] +/- number register_timestamp_add_sub(registry); - // convert_timezone( target_timezone, date, src_timezone) + // convert_timezone( target_timezone, 'timestamp') register_convert_timezone(registry); } @@ -144,6 +144,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( |target_tz, src_timestamp, output, ctx| { + if let Some(validity) = &ctx.validity { + if !validity.get_bit(output.len()) { + output.push(0); + return; + } + } // Convert source timestamp from source timezone to target timezone let p_src_timestamp = src_timestamp.to_timestamp(ctx.func_ctx.tz.tz); let src_dst_from_utc = p_src_timestamp.offset().fix().local_minus_utc(); @@ -163,10 +169,14 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { .offset() .fix() .local_minus_utc(); - let offset_as_micros_sec = - (target_dst_from_utc - src_dst_from_utc) as i64 * MICROS_PER_SEC; - let res = result_timestamp.checked_add(offset_as_micros_sec).unwrap(); - output.push(res) + let offset_as_micros_sec = (target_dst_from_utc - src_dst_from_utc) as i64; + match offset_as_micros_sec.checked_mul(MICROS_PER_SEC) { + Some(offset) => match result_timestamp.checked_add(offset) { + Some(res) => output.push(res), + None => ctx.set_error(output.len(), "calc final time error".to_string()), + }, + None => ctx.set_error(output.len(), "calc time offset error".to_string()), + } }, ), ); diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index 7d4cf1b16a970..4629201713f4e 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -1224,6 +1224,8 @@ Functions overloads: 26 contains(Array(Boolean), Boolean) :: Boolean 27 contains(Array(Boolean) NULL, Boolean NULL) :: Boolean NULL 28 contains(Array(T0) NULL, T0) :: Boolean +0 convert_timezone(String, Timestamp) :: Timestamp +1 convert_timezone(String NULL, Timestamp NULL) :: Timestamp NULL 0 cos(Float64) :: Float64 1 cos(Float64 NULL) :: Float64 NULL 0 cosine_distance(Array(Float32), Array(Float32)) :: Float32 diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test index f31d1fce57265..57b4984d6740a 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes_tz.test @@ -782,18 +782,22 @@ statement ok set timezone='UTC'; statement ok -create or replace table t(c timestamp); +create or replace table t(a string null, c timestamp null); statement ok -insert into t values('1970-01-01 00:00:00'), ('2024-10-31 22:21:15'); +insert into t values('America/Los_Angeles','1970-01-01 00:00:00'), ('America/Los_Angeles','2024-10-31 22:21:15'), (null, '1970-01-01 00:00:00'), ('Asia/Shanghai', '1970-01-01 00:00:00'), ('Asia/Shanghai', '2024-10-31 22:21:15'),('Asia/Shanghai', null), (null, null); # UTC to America/Los_Angeles, 20240-03-10 ~ 2024-11-03 is UTC-7(dst), other is UTC-8 query T -select c, CONVERT_TIMEZONE('America/Los_Angeles', c) from t; +select a, c, CONVERT_TIMEZONE(a, c) from t order by a,c; ---- -1970-01-01 00:00:00.000000 1969-12-31 16:00:00.000000 -2024-10-31 22:21:15.000000 2024-10-31 15:21:15.000000 - +America/Los_Angeles 1970-01-01 00:00:00.000000 1969-12-31 16:00:00.000000 +America/Los_Angeles 2024-10-31 22:21:15.000000 2024-10-31 15:21:15.000000 +Asia/Shanghai 1970-01-01 00:00:00.000000 1970-01-01 08:00:00.000000 +Asia/Shanghai 2024-10-31 22:21:15.000000 2024-11-01 06:21:15.000000 +Asia/Shanghai NULL NULL +NULL 1970-01-01 00:00:00.000000 NULL +NULL NULL NULL statement ok set timezone='Asia/Shanghai'; From 769e9f51a1ee0798334639c76ba426f0e9e920ba Mon Sep 17 00:00:00 2001 From: TCeason Date: Sat, 2 Nov 2024 00:44:01 +0800 Subject: [PATCH 64/64] if error output push default --- src/query/functions/src/scalars/datetime.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 2b595d8021d13..ca3fe2d84d554 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -156,10 +156,12 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { let t_tz: Tz = match target_tz.parse() { Ok(tz) => tz, Err(e) => { - return ctx.set_error( + ctx.set_error( output.len(), format!("cannot parse target `timezone`. {}", e), ); + output.push(0); + return; } }; @@ -173,9 +175,15 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { match offset_as_micros_sec.checked_mul(MICROS_PER_SEC) { Some(offset) => match result_timestamp.checked_add(offset) { Some(res) => output.push(res), - None => ctx.set_error(output.len(), "calc final time error".to_string()), + None => { + ctx.set_error(output.len(), "calc final time error".to_string()); + output.push(0); + } }, - None => ctx.set_error(output.len(), "calc time offset error".to_string()), + None => { + ctx.set_error(output.len(), "calc time offset error".to_string()); + output.push(0); + } } }, ),