Skip to content

Commit fd4ca58

Browse files
gridnevvvitshnikd
gridnevvvit
and
shnikd
authored
Merge default values for columns fixes and changes to stable 24 1 (#2140)
Co-authored-by: Nikolay Shumkov <[email protected]>
1 parent 7d89162 commit fd4ca58

38 files changed

+741
-77
lines changed

‎ydb/core/kqp/common/kqp_resolve.h

+6-4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> {
3333
ETableKind TableKind = ETableKind::Unknown;
3434
THashMap<TString, TString> Sequences;
3535
THashMap<TString, Ydb::TypedValue> DefaultFromLiteral;
36+
bool IsBuildInProgress = false;
3637

3738
TTableConstInfo() {}
3839
TTableConstInfo(const TString& path) : Path(path) {}
@@ -52,6 +53,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> {
5253
NPg::TypeDescFromPgTypeName(phyColumn.GetPgTypeName()));
5354
}
5455
column.NotNull = phyColumn.GetNotNull();
56+
column.IsBuildInProgress = phyColumn.GetIsBuildInProgress();
5557

5658
Columns.emplace(phyColumn.GetId().GetName(), std::move(column));
5759
if (!phyColumn.GetDefaultFromSequence().empty()) {
@@ -144,19 +146,19 @@ class TKqpTableKeys {
144146
const TMap<TString, NSharding::TShardingBase::TColumn>& GetColumns() const {
145147
return TableConstInfo->Columns;
146148
}
147-
149+
148150
const TVector<TString>& GetKeyColumns() const {
149151
return TableConstInfo->KeyColumns;
150152
}
151-
153+
152154
const TVector<NScheme::TTypeInfo>& GetKeyColumnTypes() const {
153155
return TableConstInfo->KeyColumnTypes;
154156
}
155-
157+
156158
const ETableKind& GetTableKind() const {
157159
return TableConstInfo->TableKind;
158160
}
159-
161+
160162
const THashMap<TString, TString>& GetSequences() const {
161163
return TableConstInfo->Sequences;
162164
}

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,8 @@ TTableMetadataResult GetTableMetadataResult(const NSchemeCache::TSchemeCacheNavi
192192
columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType, columnDesc.PTypeMod,
193193
columnDesc.DefaultFromSequence,
194194
defaultKind,
195-
columnDesc.DefaultFromLiteral
195+
columnDesc.DefaultFromLiteral,
196+
columnDesc.IsBuildInProgress
196197
)
197198
);
198199
if (columnDesc.KeyOrder >= 0) {

‎ydb/core/kqp/opt/kqp_opt_kql.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ TExprBase BuildRowsToDelete(const TKikimrTableDescription& tableData, bool withS
445445
const TPositionHandle pos, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx)
446446
{
447447
const auto tableMeta = BuildTableMeta(tableData, pos, ctx);
448-
const auto tableColumns = BuildColumnsList(tableData, pos, ctx, withSystemColumns);
448+
const auto tableColumns = BuildColumnsList(tableData, pos, ctx, withSystemColumns, true /*ignoreWriteOnlyColumns*/);
449449

450450
const auto allRows = BuildReadTable(tableColumns, pos, tableData, false, ctx, kqpCtx);
451451

@@ -517,7 +517,7 @@ TExprBase BuildDeleteTableWithIndex(const TKiDeleteTable& del, const TKikimrTabl
517517
TExprBase BuildRowsToUpdate(const TKikimrTableDescription& tableData, bool withSystemColumns, const TCoLambda& filter,
518518
const TPositionHandle pos, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx)
519519
{
520-
auto kqlReadTable = BuildReadTable(BuildColumnsList(tableData, pos, ctx, withSystemColumns), pos, tableData, false, ctx, kqpCtx);
520+
auto kqlReadTable = BuildReadTable(BuildColumnsList(tableData, pos, ctx, withSystemColumns, true /*ignoreWriteOnlyColumns*/), pos, tableData, false, ctx, kqpCtx);
521521

522522
return Build<TCoFilter>(ctx, pos)
523523
.Input(kqlReadTable)

‎ydb/core/kqp/provider/yql_kikimr_datasink.cpp

+43-4
Original file line numberDiff line numberDiff line change
@@ -671,7 +671,7 @@ class TKikimrDataSink : public TDataProviderBase
671671
TString(dataSink.Cluster()),
672672
key.GetTablePath(), node->Pos(), ctx);
673673

674-
returningColumns = BuildColumnsList(*table, node->Pos(), ctx, sysColumnsEnabled);
674+
returningColumns = BuildColumnsList(*table, node->Pos(), ctx, sysColumnsEnabled, true /*ignoreWriteOnlyColumns*/);
675675

676676
break;
677677
} else {
@@ -787,7 +787,7 @@ class TKikimrDataSink : public TDataProviderBase
787787
auto replaceIfExists = (settings.Mode.Cast().Value() == "create_or_replace");
788788
auto existringOk = (settings.Mode.Cast().Value() == "create_if_not_exists");
789789

790-
return Build<TKiCreateTable>(ctx, node->Pos())
790+
auto createTable = Build<TKiCreateTable>(ctx, node->Pos())
791791
.World(node->Child(0))
792792
.DataSink(node->Child(1))
793793
.Table().Build(key.GetTablePath())
@@ -807,8 +807,47 @@ class TKikimrDataSink : public TDataProviderBase
807807
.ExistingOk<TCoAtom>()
808808
.Value(existringOk)
809809
.Build()
810-
.Done()
811-
.Ptr();
810+
.Done();
811+
812+
bool exprEvalNeeded = false;
813+
814+
for(auto item: createTable.Cast<TKiCreateTable>().Columns()) {
815+
auto columnTuple = item.Cast<TExprList>();
816+
if (columnTuple.Size() > 2) {
817+
const auto& columnConstraints = columnTuple.Item(2).Cast<TCoNameValueTuple>();
818+
for(const auto& constraint: columnConstraints.Value().Cast<TCoNameValueTupleList>()) {
819+
if (constraint.Name().Value() != "default")
820+
continue;
821+
822+
YQL_ENSURE(constraint.Value().IsValid());
823+
bool shouldEvaluate = (
824+
constraint.Value().Cast().Ptr()->IsCallable() &&
825+
(constraint.Value().Cast().Ptr()->Content() == "PgCast") &&
826+
(constraint.Value().Cast().Ptr()->ChildrenSize() >= 1) &&
827+
(constraint.Value().Cast().Ptr()->Child(0)->IsCallable()) &&
828+
(constraint.Value().Cast().Ptr()->Child(0)->Content() == "PgConst")
829+
);
830+
831+
if (shouldEvaluate) {
832+
auto evaluatedExpr = ctx.Builder(constraint.Value().Cast().Ptr()->Pos())
833+
.Callable("EvaluateExpr")
834+
.Add(0, constraint.Value().Cast().Ptr())
835+
.Seal()
836+
.Build();
837+
838+
constraint.Ptr()->ChildRef(TCoNameValueTuple::idx_Value) = evaluatedExpr;
839+
exprEvalNeeded = true;
840+
}
841+
}
842+
}
843+
}
844+
845+
if (exprEvalNeeded) {
846+
ctx.Step.Repeat(TExprStep::ExprEval);
847+
}
848+
849+
return createTable.Ptr();
850+
812851
} else if (mode == "alter") {
813852
for (auto setting : settings.Other) {
814853
if (setting.Name().Value() == "intent") {

‎ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ TCoAtomList TKiReadTable::GetSelectColumns(TExprContext& ctx, const TKikimrTable
1717
bool withSystemColumns) const
1818
{
1919
if (Select().Maybe<TCoVoid>()) {
20-
return BuildColumnsList(tableData, Pos(), ctx, withSystemColumns);
20+
return BuildColumnsList(tableData, Pos(), ctx, withSystemColumns, true /*ignoreWriteOnlyColumns*/);
2121
}
2222

2323
return Select().Cast<TCoAtomList>();

‎ydb/core/kqp/provider/yql_kikimr_gateway.h

+5-1
Original file line numberDiff line numberDiff line change
@@ -215,13 +215,14 @@ struct TKikimrColumnMetadata {
215215
NKikimrKqp::TKqpColumnMetadataProto::EDefaultKind DefaultKind = NKikimrKqp::TKqpColumnMetadataProto::DEFAULT_KIND_UNSPECIFIED;
216216
TString DefaultFromSequence;
217217
Ydb::TypedValue DefaultFromLiteral;
218+
bool IsBuildInProgress = false;
218219

219220
TKikimrColumnMetadata() = default;
220221

221222
TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull,
222223
NKikimr::NScheme::TTypeInfo typeInfo = {}, const TString& typeMod = {}, const TString& defaultFromSequence = {},
223224
NKikimrKqp::TKqpColumnMetadataProto::EDefaultKind defaultKind = NKikimrKqp::TKqpColumnMetadataProto::DEFAULT_KIND_UNSPECIFIED,
224-
const Ydb::TypedValue& defaultFromLiteral = {})
225+
const Ydb::TypedValue& defaultFromLiteral = {}, bool isBuildInProgress = false)
225226
: Name(name)
226227
, Id(id)
227228
, Type(type)
@@ -231,6 +232,7 @@ struct TKikimrColumnMetadata {
231232
, DefaultKind(defaultKind)
232233
, DefaultFromSequence(defaultFromSequence)
233234
, DefaultFromLiteral(defaultFromLiteral)
235+
, IsBuildInProgress(isBuildInProgress)
234236
{}
235237

236238
explicit TKikimrColumnMetadata(const NKikimrKqp::TKqpColumnMetadataProto* message)
@@ -242,6 +244,7 @@ struct TKikimrColumnMetadata {
242244
, DefaultKind(message->GetDefaultKind())
243245
, DefaultFromSequence(message->GetDefaultFromSequence())
244246
, DefaultFromLiteral(message->GetDefaultFromLiteral())
247+
, IsBuildInProgress(message->GetIsBuildInProgress())
245248
{
246249
auto typeInfoMod = NKikimr::NScheme::TypeInfoModFromProtoColumnType(message->GetTypeId(),
247250
message->HasTypeInfo() ? &message->GetTypeInfo() : nullptr);
@@ -279,6 +282,7 @@ struct TKikimrColumnMetadata {
279282
message->SetDefaultFromSequence(DefaultFromSequence);
280283
message->SetDefaultKind(DefaultKind);
281284
message->MutableDefaultFromLiteral()->CopyFrom(DefaultFromLiteral);
285+
message->SetIsBuildInProgress(IsBuildInProgress);
282286
if (columnType.TypeInfo) {
283287
*message->MutableTypeInfo() = *columnType.TypeInfo;
284288
}

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

+5-1
Original file line numberDiff line numberDiff line change
@@ -463,10 +463,14 @@ bool TKikimrKey::Extract(const TExprNode& key) {
463463
}
464464

465465
TCoAtomList BuildColumnsList(const TKikimrTableDescription& table, TPositionHandle pos,
466-
TExprContext& ctx, bool withSystemColumns)
466+
TExprContext& ctx, bool withSystemColumns, bool ignoreWriteOnlyColumns)
467467
{
468468
TVector<TExprBase> columnsToSelect;
469469
for (const auto& pair : table.Metadata->Columns) {
470+
if (pair.second.IsBuildInProgress && ignoreWriteOnlyColumns) {
471+
continue;
472+
}
473+
470474
auto atom = Build<TCoAtom>(ctx, pos)
471475
.Value(pair.second.Name)
472476
.Done();

‎ydb/core/kqp/provider/yql_kikimr_provider_impl.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ TAutoPtr<IGraphTransformer> CreateKiSinkCallableExecutionTransformer(
209209
TAutoPtr<IGraphTransformer> CreateKiSinkPlanInfoTransformer(TIntrusivePtr<IKikimrQueryExecutor> queryExecutor);
210210

211211
NNodes::TCoAtomList BuildColumnsList(const TKikimrTableDescription& table, TPositionHandle pos,
212-
TExprContext& ctx, bool withSystemColumns);
212+
TExprContext& ctx, bool withSystemColumns, bool ignoreWriteOnlyColumns);
213213

214214
const TTypeAnnotationNode* GetReadTableRowType(TExprContext& ctx, const TKikimrTablesData& tablesData,
215215
const TString& cluster, const TString& table, NNodes::TCoAtomList select, bool withSystemColumns = false);

‎ydb/core/kqp/provider/yql_kikimr_type_ann.cpp

+49-14
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,13 @@ class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer
418418
THashSet<TString> generateColumnsIfInsertColumnsSet;
419419

420420
for(const auto& [name, info] : table->Metadata->Columns) {
421+
if (info.IsBuildInProgress && rowType->FindItem(name)) {
422+
ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder()
423+
<< "Column is under build operation, write operation is not allowed to column: " << name
424+
<< " for table: " << table->Metadata->Name));
425+
return TStatus::Error;
426+
}
427+
421428
if (rowType->FindItem(name)) {
422429
continue;
423430
}
@@ -431,7 +438,7 @@ class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer
431438
}
432439

433440
if (info.IsDefaultKindDefined()) {
434-
if (op == TYdbOperation::Upsert) {
441+
if (op == TYdbOperation::Upsert && !info.IsBuildInProgress) {
435442
generateColumnsIfInsertColumnsSet.emplace(name);
436443
}
437444

@@ -625,6 +632,13 @@ class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer
625632
<< "Column '" << item->GetName() << "' does not exist in table '" << node.Table().Value() << "'."));
626633
return TStatus::Error;
627634
}
635+
636+
if (column->IsBuildInProgress) {
637+
ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder()
638+
<< "Column '" << item->GetName() << "' is under the build operation '" << node.Table().Value() << "'."));
639+
return TStatus::Error;
640+
}
641+
628642
if (column->NotNull && item->HasOptionalOrNull()) {
629643
if (item->GetItemType()->GetKind() == ETypeAnnotationKind::Pg) {
630644
//no type-level notnull check for pg types.
@@ -636,6 +650,8 @@ class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer
636650
}
637651
}
638652

653+
654+
639655
if (!node.ReturningColumns().Empty()) {
640656
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder()
641657
<< "It is not allowed to use returning"));
@@ -808,7 +824,13 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over
808824

809825
columnMeta.SetDefaultFromLiteral();
810826

811-
if (auto pgConst = constraint.Value().Maybe<TCoPgConst>()) {
827+
YQL_ENSURE(constraint.Value().IsValid());
828+
const auto& constrValue = constraint.Value().Cast();
829+
bool isPgNull = constrValue.Ptr()->IsCallable() &&
830+
constrValue.Ptr()->Content() == "PgCast" && constrValue.Ptr()->ChildrenSize() >= 1 &&
831+
constrValue.Ptr()->Child(0)->IsCallable() && constrValue.Ptr()->Child(0)->Content() == "Null";
832+
833+
if (constrValue.Maybe<TCoPgConst>() || isPgNull) {
812834
auto actualPgType = actualType->Cast<TPgExprType>();
813835
YQL_ENSURE(actualPgType);
814836

@@ -819,25 +841,38 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over
819841
return TStatus::Error;
820842
}
821843

822-
TString content = TString(pgConst.Cast().Value().Value());
823-
auto parseResult = NKikimr::NPg::PgNativeBinaryFromNativeText(content, typeDesc);
824-
if (parseResult.Error) {
825-
ctx.AddError(TIssue(ctx.GetPosition(constraint.Pos()),
826-
TStringBuilder() << "Failed to parse default expr for typename " << actualPgType->GetName()
827-
<< ", error reason: " << *parseResult.Error));
828-
return TStatus::Error;
844+
if (isPgNull) {
845+
if (columnMeta.NotNull) {
846+
ctx.AddError(TIssue(ctx.GetPosition(constraint.Pos()), TStringBuilder() << "Default expr " << columnName
847+
<< " is nullable or optional, but column has not null constraint. "));
848+
return TStatus::Error;
849+
}
850+
851+
columnMeta.DefaultFromLiteral.mutable_value()->set_null_flag_value(NProtoBuf::NULL_VALUE);
852+
853+
} else {
854+
YQL_ENSURE(constrValue.Maybe<TCoPgConst>());
855+
auto pgConst = constrValue.Cast<TCoPgConst>();
856+
TString content = TString(pgConst.Value().Value());
857+
auto parseResult = NKikimr::NPg::PgNativeBinaryFromNativeText(content, typeDesc);
858+
if (parseResult.Error) {
859+
ctx.AddError(TIssue(ctx.GetPosition(constraint.Pos()),
860+
TStringBuilder() << "Failed to parse default expr for typename " << actualPgType->GetName()
861+
<< ", error reason: " << *parseResult.Error));
862+
return TStatus::Error;
863+
}
864+
865+
columnMeta.DefaultFromLiteral.mutable_value()->set_bytes_value(parseResult.Str);
829866
}
830867

831-
columnMeta.DefaultFromLiteral.mutable_value()->set_bytes_value(parseResult.Str);
832868
auto* pg = columnMeta.DefaultFromLiteral.mutable_type()->mutable_pg_type();
833-
834869
pg->set_type_name(NKikimr::NPg::PgTypeNameFromTypeDesc(typeDesc));
835870
pg->set_oid(NKikimr::NPg::PgTypeIdFromTypeDesc(typeDesc));
836-
} else if (auto literal = constraint.Value().Maybe<TCoDataCtor>()) {
837-
FillLiteralProto(constraint.Value().Cast<TCoDataCtor>(), columnMeta.DefaultFromLiteral);
871+
} else if (auto literal = constrValue.Maybe<TCoDataCtor>()) {
872+
FillLiteralProto(literal.Cast(), columnMeta.DefaultFromLiteral);
838873
} else {
839874
ctx.AddError(TIssue(ctx.GetPosition(constraint.Pos()),
840-
TStringBuilder() << "Unsupported type of default value " << constraint.Value().Cast().Ptr()->Content()));
875+
TStringBuilder() << "Unsupported type of default value"));
841876
return TStatus::Error;
842877
}
843878

‎ydb/core/kqp/query_compiler/kqp_query_compiler.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ void FillTable(const TKikimrTableMetadata& tableMeta, THashSet<TStringBuf>&& col
174174
phyColumn.MutableId()->SetId(column->Id);
175175
phyColumn.MutableId()->SetName(column->Name);
176176
phyColumn.SetTypeId(column->TypeInfo.GetTypeId());
177+
phyColumn.SetIsBuildInProgress(column->IsBuildInProgress);
177178
if (column->IsDefaultFromSequence()) {
178179
phyColumn.SetDefaultFromSequence(column->DefaultFromSequence);
179180
} else if (column->IsDefaultFromLiteral()) {

‎ydb/core/kqp/ut/pg/kqp_pg_ut.cpp

+43
Original file line numberDiff line numberDiff line change
@@ -3403,7 +3403,50 @@ Y_UNIT_TEST_SUITE(KqpPg) {
34033403
)", FormatResultSetYson(result.GetResultSet(0)));
34043404
}
34053405
}
3406+
/*
3407+
Y_UNIT_TEST(InsertValuesFromTableWithDefaultAndCast) {
3408+
NKikimrConfig::TAppConfig appConfig;
3409+
appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true);
3410+
auto setting = NKikimrKqp::TKqpSetting();
3411+
auto serverSettings = TKikimrSettings()
3412+
.SetAppConfig(appConfig)
3413+
.SetKqpSettings({setting});
3414+
TKikimrRunner kikimr(serverSettings.SetWithSampleTables(false));
3415+
auto db = kikimr.GetQueryClient();
3416+
auto settings = NYdb::NQuery::TExecuteQuerySettings().Syntax(NYdb::NQuery::ESyntax::Pg);
3417+
{
3418+
auto result = db.ExecuteQuery(R"(
3419+
CREATE TABLE t (
3420+
a INT,
3421+
b int DEFAULT 5::int4,
3422+
c int DEFAULT '7'::int4,
3423+
d varchar(20) DEFAULT 'foo'::varchar(2),
3424+
e int DEFAULT NULL,
3425+
f bit varying(5) DEFAULT '1001',
3426+
PRIMARY KEY(a)
3427+
);
3428+
)", NYdb::NQuery::TTxControl::NoTx(), settings).ExtractValueSync();
3429+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
3430+
}
3431+
{
3432+
auto result = db.ExecuteQuery(R"(
3433+
INSERT INTO t VALUES(1);
3434+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
3435+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
3436+
}
3437+
{
3438+
auto result = db.ExecuteQuery(R"(
3439+
SELECT * FROM t;
3440+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
3441+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
34063442
3443+
UNIT_ASSERT_C(!result.GetResultSets().empty(), "results are empty");
3444+
CompareYson(R"(
3445+
[["1";"5";"7";"fo";#;"1001"]]
3446+
)", FormatResultSetYson(result.GetResultSet(0)));
3447+
}
3448+
}
3449+
*/
34073450
Y_UNIT_TEST(InsertValuesFromTableWithDefaultBool) {
34083451
NKikimrConfig::TAppConfig appConfig;
34093452
appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true);

0 commit comments

Comments
 (0)