Skip to content

Commit 11b0174

Browse files
gridnevvvitdcherednik
gridnevvvit
and
dcherednik
authored
Merge default related changes to stable-24-1 (#4334)
Co-authored-by: Daniil Cherednik <[email protected]>
1 parent ec017a1 commit 11b0174

8 files changed

+578
-179
lines changed

‎ydb/core/kqp/provider/yql_kikimr_datasink.cpp

Lines changed: 125 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,114 @@ class TKikimrDataSink : public TDataProviderBase
511511
return false;
512512
}
513513

514+
bool IsEvaluatedDefaultNode(TExprNode::TPtr input) {
515+
auto node = TExprBase(input);
516+
517+
if (node.Maybe<TCoNothing>()) {
518+
return true;
519+
}
520+
521+
if (input->IsCallable()
522+
&& input->Content() == "PgCast"
523+
&& input->ChildrenSize() >= 1
524+
&& TExprBase(input->Child(0)).Maybe<TCoNull>())
525+
{
526+
return true;
527+
}
528+
529+
if (node.Maybe<TCoAtom>()) {
530+
return true;
531+
}
532+
533+
if (node.Maybe<TCoNull>()) {
534+
return true;
535+
}
536+
537+
if (auto maybeJust = node.Maybe<TCoJust>()) {
538+
return IsEvaluatedDefaultNode(maybeJust.Cast().Input().Ptr());
539+
}
540+
541+
if (node.Maybe<TCoPgConst>()) {
542+
return true;
543+
}
544+
545+
if (node.Maybe<TCoDataCtor>()) {
546+
return true;
547+
}
548+
549+
if (node.Maybe<TCoDataType>()) {
550+
return true;
551+
}
552+
553+
if (node.Maybe<TCoPgType>()) {
554+
return true;
555+
}
556+
557+
return false;
558+
}
559+
560+
bool CanEvaluateDefaultValue(TExprNode::TPtr input) {
561+
if (IsEvaluatedDefaultNode(input)) {
562+
return true;
563+
}
564+
565+
static const THashSet<TString> allowedNodes = {
566+
"Concat", "Just", "Optional", "SafeCast", "AsList",
567+
"+", "-", "*", "/", "%", "PgCast"};
568+
569+
if (input->IsCallable("PgCast") && input->Child(0)->IsCallable("PgConst")) {
570+
return true;
571+
}
572+
573+
if (input->IsCallable(allowedNodes)) {
574+
for (size_t i = 0; i < input->ChildrenSize(); i++) {
575+
auto callableInput = input->Child(i);
576+
if (!CanEvaluateDefaultValue(callableInput)) {
577+
return false;
578+
}
579+
}
580+
return true;
581+
}
582+
583+
return false;
584+
}
585+
586+
bool ShouldEvaluateDefaultValue(TExprNode::TPtr input) {
587+
if (IsEvaluatedDefaultNode(input)) {
588+
return false;
589+
}
590+
591+
return CanEvaluateDefaultValue(input);
592+
}
593+
594+
bool EvaluateDefaultValuesIfNeeded(TExprContext& ctx, TExprList columns) {
595+
bool exprEvalNeeded = false;
596+
for(auto item: columns) {
597+
auto columnTuple = item.Cast<TExprList>();
598+
if (columnTuple.Size() > 2) {
599+
const auto& columnConstraints = columnTuple.Item(2).Cast<TCoNameValueTuple>();
600+
for(const auto& constraint: columnConstraints.Value().Cast<TCoNameValueTupleList>()) {
601+
if (constraint.Name().Value() != "default")
602+
continue;
603+
604+
YQL_ENSURE(constraint.Value().IsValid());
605+
606+
if (ShouldEvaluateDefaultValue(constraint.Value().Cast().Ptr())) {
607+
auto evaluatedExpr = ctx.Builder(constraint.Value().Cast().Ptr()->Pos())
608+
.Callable("EvaluateExpr")
609+
.Add(0, constraint.Value().Cast().Ptr())
610+
.Seal()
611+
.Build();
612+
613+
constraint.Ptr()->ChildRef(TCoNameValueTuple::idx_Value) = evaluatedExpr;
614+
exprEvalNeeded = true;
615+
}
616+
}
617+
}
618+
}
619+
return exprEvalNeeded;
620+
}
621+
514622
static TExprNode::TPtr MakeKiDropTable(const TExprNode::TPtr& node, const NCommon::TWriteTableSettings& settings,
515623
const TKikimrKey& key, TExprContext& ctx)
516624
{
@@ -809,39 +917,7 @@ class TKikimrDataSink : public TDataProviderBase
809917
.Build()
810918
.Done();
811919

812-
bool exprEvalNeeded = false;
813-
814-
for(auto item: createTable.Cast<TKiCreateTable>().Columns()) {
815-
auto columnTuple = item.Cast<TExprList>();
816-
if (columnTuple.Size() > 2) {
817-
const auto& columnConstraints = columnTuple.Item(2).Cast<TCoNameValueTuple>();
818-
for(const auto& constraint: columnConstraints.Value().Cast<TCoNameValueTupleList>()) {
819-
if (constraint.Name().Value() != "default")
820-
continue;
821-
822-
YQL_ENSURE(constraint.Value().IsValid());
823-
bool shouldEvaluate = (
824-
constraint.Value().Cast().Ptr()->IsCallable() &&
825-
(constraint.Value().Cast().Ptr()->Content() == "PgCast") &&
826-
(constraint.Value().Cast().Ptr()->ChildrenSize() >= 1) &&
827-
(constraint.Value().Cast().Ptr()->Child(0)->IsCallable()) &&
828-
(constraint.Value().Cast().Ptr()->Child(0)->Content() == "PgConst")
829-
);
830-
831-
if (shouldEvaluate) {
832-
auto evaluatedExpr = ctx.Builder(constraint.Value().Cast().Ptr()->Pos())
833-
.Callable("EvaluateExpr")
834-
.Add(0, constraint.Value().Cast().Ptr())
835-
.Seal()
836-
.Build();
837-
838-
constraint.Ptr()->ChildRef(TCoNameValueTuple::idx_Value) = evaluatedExpr;
839-
exprEvalNeeded = true;
840-
}
841-
}
842-
}
843-
}
844-
920+
bool exprEvalNeeded = EvaluateDefaultValuesIfNeeded(ctx, createTable.Cast<TKiCreateTable>().Columns());
845921
if (exprEvalNeeded) {
846922
ctx.Step.Repeat(TExprStep::ExprEval);
847923
}
@@ -858,15 +934,27 @@ class TKikimrDataSink : public TDataProviderBase
858934

859935
YQL_ENSURE(settings.AlterActions);
860936
YQL_ENSURE(!settings.AlterActions.Cast().Empty());
861-
862-
return Build<TKiAlterTable>(ctx, node->Pos())
937+
auto alterTable = Build<TKiAlterTable>(ctx, node->Pos())
863938
.World(node->Child(0))
864939
.DataSink(node->Child(1))
865940
.Table().Build(key.GetTablePath())
866941
.Actions(settings.AlterActions.Cast())
867942
.TableType(tableType)
868-
.Done()
869-
.Ptr();
943+
.Done();
944+
945+
bool exprEvalNeeded = false;
946+
for (const auto& action : alterTable.Actions()) {
947+
auto name = action.Name().Value();
948+
if (name == "addColumns") {
949+
exprEvalNeeded |= EvaluateDefaultValuesIfNeeded(ctx, action.Value().Cast<TExprList>());
950+
}
951+
}
952+
953+
if (exprEvalNeeded) {
954+
ctx.Step.Repeat(TExprStep::ExprEval);
955+
}
956+
957+
return alterTable.Ptr();
870958

871959
} else if (mode == "drop" || mode == "drop_if_exists") {
872960
return MakeKiDropTable(node, settings, key, ctx);

‎ydb/core/kqp/provider/yql_kikimr_exec.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1104,8 +1104,14 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKi
11041104
}
11051105

11061106
columnBuild->SetColumnName(TString(columnName));
1107-
FillLiteralProto(constraint.Value().Cast<TCoDataCtor>(), *columnBuild->mutable_default_from_literal());
1107+
auto err = FillLiteralProto(constraint.Value().Cast(), actualType, *columnBuild->mutable_default_from_literal());
1108+
if (err) {
1109+
ctx.AddError(TIssue(ctx.GetPosition(constraint.Pos()), err.value()));
1110+
return SyncError();
1111+
}
1112+
11081113
hasDefaultValue = true;
1114+
11091115
} else if (constraint.Name().Value() == "not_null") {
11101116
if (columnBuild == nullptr) {
11111117
columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column();

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

Lines changed: 74 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TKiOperationList& o
531531
return protoOps;
532532
}
533533

534+
// Is used only for key types now
534535
template<typename TProto>
535536
void FillLiteralProtoImpl(const NNodes::TCoDataCtor& literal, TProto& proto) {
536537
auto type = literal.Ref().GetTypeAnn();
@@ -562,6 +563,7 @@ void FillLiteralProtoImpl(const NNodes::TCoDataCtor& literal, TProto& proto) {
562563
case EDataSlot::Datetime:
563564
protoValue.SetUint32(FromString<ui32>(value));
564565
break;
566+
case EDataSlot::Int8:
565567
case EDataSlot::Int32:
566568
protoValue.SetInt32(FromString<i32>(value));
567569
break;
@@ -578,8 +580,18 @@ void FillLiteralProtoImpl(const NNodes::TCoDataCtor& literal, TProto& proto) {
578580
protoValue.SetBytes(value.Data(), value.Size());
579581
break;
580582
case EDataSlot::Utf8:
583+
case EDataSlot::Json:
581584
protoValue.SetText(ToString(value));
582585
break;
586+
case EDataSlot::Double:
587+
protoValue.SetDouble(FromString<double>(value));
588+
break;
589+
case EDataSlot::Float:
590+
protoValue.SetFloat(FromString<float>(value));
591+
break;
592+
case EDataSlot::Yson:
593+
protoValue.SetBytes(ToString(value));
594+
break;
583595
case EDataSlot::Decimal: {
584596
const auto paramsDataType = type->Cast<TDataExprParamsType>();
585597
auto precision = FromString<ui8>(paramsDataType->GetParamOne());
@@ -603,8 +615,56 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, NKqpProto::TKqpPhyLite
603615
FillLiteralProtoImpl(literal, proto);
604616
}
605617

606-
void FillLiteralProto(const NNodes::TCoDataCtor& literal, NKikimrMiniKQL::TResult& proto) {
607-
FillLiteralProtoImpl(literal, proto);
618+
bool IsPgNullExprNode(const NNodes::TExprBase& maybeLiteral) {
619+
return maybeLiteral.Ptr()->IsCallable() &&
620+
maybeLiteral.Ptr()->Content() == "PgCast" && maybeLiteral.Ptr()->ChildrenSize() >= 1 &&
621+
maybeLiteral.Ptr()->Child(0)->IsCallable() && maybeLiteral.Ptr()->Child(0)->Content() == "Null";
622+
}
623+
624+
std::optional<TString> FillLiteralProto(NNodes::TExprBase maybeLiteral, const TTypeAnnotationNode* valueType, Ydb::TypedValue& proto)
625+
{
626+
if (auto maybeJust = maybeLiteral.Maybe<TCoJust>()) {
627+
maybeLiteral = maybeJust.Cast().Input();
628+
}
629+
630+
if (auto literal = maybeLiteral.Maybe<TCoDataCtor>()) {
631+
FillLiteralProto(literal.Cast(), proto);
632+
return std::nullopt;
633+
}
634+
635+
const bool isPgNull = IsPgNullExprNode(maybeLiteral);
636+
if (maybeLiteral.Maybe<TCoPgConst>() || isPgNull) {
637+
YQL_ENSURE(valueType);
638+
auto actualPgType = valueType->Cast<TPgExprType>();
639+
YQL_ENSURE(actualPgType);
640+
641+
auto* typeDesc = NKikimr::NPg::TypeDescFromPgTypeId(actualPgType->GetId());
642+
if (!typeDesc) {
643+
return TStringBuilder() << "Failed to parse default expr typename " << actualPgType->GetName();
644+
}
645+
646+
if (isPgNull) {
647+
proto.mutable_value()->set_null_flag_value(NProtoBuf::NULL_VALUE);
648+
} else {
649+
YQL_ENSURE(maybeLiteral.Maybe<TCoPgConst>());
650+
auto pgConst = maybeLiteral.Cast<TCoPgConst>();
651+
TString content = TString(pgConst.Value().Value());
652+
auto parseResult = NKikimr::NPg::PgNativeBinaryFromNativeText(content, typeDesc);
653+
if (parseResult.Error) {
654+
return TStringBuilder() << "Failed to parse default expr for typename " << actualPgType->GetName()
655+
<< ", error reason: " << *parseResult.Error;
656+
}
657+
658+
proto.mutable_value()->set_bytes_value(parseResult.Str);
659+
}
660+
661+
auto* pg = proto.mutable_type()->mutable_pg_type();
662+
pg->set_type_name(NKikimr::NPg::PgTypeNameFromTypeDesc(typeDesc));
663+
pg->set_oid(NKikimr::NPg::PgTypeIdFromTypeDesc(typeDesc));
664+
return std::nullopt;
665+
}
666+
667+
return TStringBuilder() << "Unsupported type of literal: " << maybeLiteral.Ptr()->Content();
608668
}
609669

610670
void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto)
@@ -617,8 +677,7 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto
617677
auto slot = type->Cast<TDataExprType>()->GetSlot();
618678
auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId;
619679

620-
YQL_ENSURE(NKikimr::NScheme::NTypeIds::IsYqlType(typeId) &&
621-
NKikimr::NSchemeShard::IsAllowedKeyType(NKikimr::NScheme::TTypeInfo(typeId)));
680+
YQL_ENSURE(NKikimr::NScheme::NTypeIds::IsYqlType(typeId));
622681

623682
auto& protoType = *proto.mutable_type();
624683
auto& protoValue = *proto.mutable_value();
@@ -637,6 +696,7 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto
637696
case EDataSlot::Datetime:
638697
protoValue.set_uint32_value(FromString<ui32>(value));
639698
break;
699+
case EDataSlot::Int8:
640700
case EDataSlot::Int32:
641701
protoValue.set_int32_value(FromString<i32>(value));
642702
break;
@@ -653,8 +713,18 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto
653713
protoValue.set_bytes_value(value.Data(), value.Size());
654714
break;
655715
case EDataSlot::Utf8:
716+
case EDataSlot::Json:
656717
protoValue.set_text_value(ToString(value));
657718
break;
719+
case EDataSlot::Double:
720+
protoValue.set_double_value(FromString<double>(value));
721+
break;
722+
case EDataSlot::Float:
723+
protoValue.set_float_value(FromString<float>(value));
724+
break;
725+
case EDataSlot::Yson:
726+
protoValue.set_bytes_value(ToString(value));
727+
break;
658728
case EDataSlot::Decimal: {
659729
const auto paramsDataType = type->Cast<TDataExprParamsType>();
660730
auto precision = FromString<ui8>(paramsDataType->GetParamOne());

‎ydb/core/kqp/provider/yql_kikimr_provider_impl.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ void TableDescriptionToTableInfo(const TKikimrTableDescription& desc, TYdbOperat
224224
void TableDescriptionToTableInfo(const TKikimrTableDescription& desc, TYdbOperation op,
225225
TVector<NKqpProto::TKqpTableInfo>& infos);
226226

227-
void FillLiteralProto(const NNodes::TCoDataCtor& literal, NKikimrMiniKQL::TResult& proto);
227+
bool IsPgNullExprNode(const NNodes::TExprBase& maybeLiteral);
228+
std::optional<TString> FillLiteralProto(NNodes::TExprBase maybeLiteral, const TTypeAnnotationNode* valueType, Ydb::TypedValue& proto);
228229
void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto);
229230
// todo gvit switch to ydb typed value.
230231
void FillLiteralProto(const NNodes::TCoDataCtor& literal, NKqpProto::TKqpPhyLiteralValue& proto);

0 commit comments

Comments
 (0)