From 908497e32fe6db2836bfe8243269e50f8bce5510 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 18:41:25 +0000 Subject: [PATCH 01/10] extract protobuf helpers from validation --- ydb/public/lib/{validation => protobuf}/helpers.cpp | 0 ydb/public/lib/{validation => protobuf}/helpers.h | 0 ydb/public/lib/protobuf/ya.make | 11 +++++++++++ ydb/public/lib/validation/main.cpp | 5 ++--- ydb/public/lib/validation/ya.make | 2 +- ydb/public/lib/ya.make | 1 + 6 files changed, 15 insertions(+), 4 deletions(-) rename ydb/public/lib/{validation => protobuf}/helpers.cpp (100%) rename ydb/public/lib/{validation => protobuf}/helpers.h (100%) create mode 100644 ydb/public/lib/protobuf/ya.make diff --git a/ydb/public/lib/validation/helpers.cpp b/ydb/public/lib/protobuf/helpers.cpp similarity index 100% rename from ydb/public/lib/validation/helpers.cpp rename to ydb/public/lib/protobuf/helpers.cpp diff --git a/ydb/public/lib/validation/helpers.h b/ydb/public/lib/protobuf/helpers.h similarity index 100% rename from ydb/public/lib/validation/helpers.h rename to ydb/public/lib/protobuf/helpers.h diff --git a/ydb/public/lib/protobuf/ya.make b/ydb/public/lib/protobuf/ya.make new file mode 100644 index 000000000000..75a4ee584d7c --- /dev/null +++ b/ydb/public/lib/protobuf/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +PEERDIR( + contrib/libs/protoc +) + +SRCS( + helpers.cpp +) + +END() diff --git a/ydb/public/lib/validation/main.cpp b/ydb/public/lib/validation/main.cpp index b21d6537952f..2d0109d2da36 100644 --- a/ydb/public/lib/validation/main.cpp +++ b/ydb/public/lib/validation/main.cpp @@ -1,7 +1,6 @@ -#include "helpers.h" - -#include #include +#include +#include #include #include diff --git a/ydb/public/lib/validation/ya.make b/ydb/public/lib/validation/ya.make index e02d0e35a252..06fcc92715b4 100644 --- a/ydb/public/lib/validation/ya.make +++ b/ydb/public/lib/validation/ya.make @@ -3,10 +3,10 @@ PROGRAM() PEERDIR( contrib/libs/protoc ydb/public/api/protos/annotations + ydb/public/lib/protobuf ) SRCS( - helpers.cpp main.cpp ) diff --git a/ydb/public/lib/ya.make b/ydb/public/lib/ya.make index 65846647a2ab..c0a04b7ed9f0 100644 --- a/ydb/public/lib/ya.make +++ b/ydb/public/lib/ya.make @@ -7,6 +7,7 @@ RECURSE( json_value jwt operation_id + protobuf scheme_types stat_visualization validation From 98ee85d12708db9b433ad2abed2c0b94c6ec9939 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 18:49:16 +0000 Subject: [PATCH 02/10] add additional helpers --- ydb/public/lib/protobuf/helpers.cpp | 15 +++++++++++++-- ydb/public/lib/protobuf/helpers.h | 6 ++++-- ydb/public/lib/validation/main.cpp | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ydb/public/lib/protobuf/helpers.cpp b/ydb/public/lib/protobuf/helpers.cpp index 2f8103b5c9f2..680b89e0b713 100644 --- a/ydb/public/lib/protobuf/helpers.cpp +++ b/ydb/public/lib/protobuf/helpers.cpp @@ -11,7 +11,7 @@ #include #include -namespace NKikimr::NValidation { +namespace NKikimr::NProtobuf { static TString ProtoFileNameStripped(const google::protobuf::Descriptor* message) { return google::protobuf::compiler::cpp::StripProto(message->file()->name()); @@ -33,6 +33,10 @@ TString NamespaceScope() { return "namespace_scope"; } +TString IncludesScope() { + return "includes"; +} + TString ClassName(const google::protobuf::Descriptor* message) { const TString ns = message->file()->package(); TString className = !ns.empty() ? message->full_name().substr(ns.size() + 1) : message->full_name(); @@ -40,6 +44,13 @@ TString ClassName(const google::protobuf::Descriptor* message) { return className; } +TString FullyQualifiedClassName(const google::protobuf::Descriptor* message) { + TString className = message->full_name(); + SubstGlobal(className, ".", "::"); + className.insert(0, "::"); + return className; +} + bool IsCustomMessage(const google::protobuf::Descriptor* message) { if (!message) { return false; @@ -73,4 +84,4 @@ bool IsCustomMessage(const google::protobuf::Descriptor* message) { return true; } -} +} // namespace NKikimr::NProtobuf diff --git a/ydb/public/lib/protobuf/helpers.h b/ydb/public/lib/protobuf/helpers.h index 8f1a6355ce1a..a598b0fa3a45 100644 --- a/ydb/public/lib/protobuf/helpers.h +++ b/ydb/public/lib/protobuf/helpers.h @@ -4,14 +4,16 @@ #include -namespace NKikimr::NValidation { +namespace NKikimr::NProtobuf { TString HeaderFileName(const google::protobuf::Descriptor* message); TString SourceFileName(const google::protobuf::Descriptor* message); TString ClassScope(const google::protobuf::Descriptor* message); TString NamespaceScope(); +TString IncludesScope(); +TString FullyQualifiedClassName(const google::protobuf::Descriptor* message); TString ClassName(const google::protobuf::Descriptor* message); bool IsCustomMessage(const google::protobuf::Descriptor* message); -} +} // namespace NKikimr::NProtobuf diff --git a/ydb/public/lib/validation/main.cpp b/ydb/public/lib/validation/main.cpp index 2d0109d2da36..3fb3be26c504 100644 --- a/ydb/public/lib/validation/main.cpp +++ b/ydb/public/lib/validation/main.cpp @@ -19,6 +19,7 @@ namespace NKikimr::NValidation { using namespace google::protobuf::compiler; using namespace google::protobuf; +using namespace NKikimr::NProtobuf; using TVariables = std::map; From 495cc4a1109d0021595ba0a697137bbd9c500497 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 18:55:00 +0000 Subject: [PATCH 03/10] add basic plugin implementation --- ydb/core/config/protos/marker.proto | 16 + ydb/core/config/protos/ya.make | 9 + .../config/tools/protobuf_plugin/main.cpp | 279 ++++++++++++++++++ ydb/core/config/tools/protobuf_plugin/ut.cpp | 24 ++ .../ut/protos/config_root_test.proto | 19 ++ .../tools/protobuf_plugin/ut/protos/ya.make | 15 + .../config/tools/protobuf_plugin/ut/ya.make | 18 ++ ydb/core/config/tools/protobuf_plugin/ya.make | 17 ++ ydb/core/config/tools/ya.make | 3 + ydb/core/config/ya.make | 2 + 10 files changed, 402 insertions(+) create mode 100644 ydb/core/config/protos/marker.proto create mode 100644 ydb/core/config/protos/ya.make create mode 100644 ydb/core/config/tools/protobuf_plugin/main.cpp create mode 100644 ydb/core/config/tools/protobuf_plugin/ut.cpp create mode 100644 ydb/core/config/tools/protobuf_plugin/ut/protos/config_root_test.proto create mode 100644 ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make create mode 100644 ydb/core/config/tools/protobuf_plugin/ut/ya.make create mode 100644 ydb/core/config/tools/protobuf_plugin/ya.make create mode 100644 ydb/core/config/tools/ya.make diff --git a/ydb/core/config/protos/marker.proto b/ydb/core/config/protos/marker.proto new file mode 100644 index 000000000000..e973f78b6e79 --- /dev/null +++ b/ydb/core/config/protos/marker.proto @@ -0,0 +1,16 @@ +import "google/protobuf/descriptor.proto"; + +package NKikimrConfig.NMarkers; +option java_package = "ru.yandex.kikimr.proto.markers"; + +extend google.protobuf.MessageOptions { + optional bool Root = 81001; + optional bool CombinedType = 81002; + optional bool WithMapType = 81003; +} + +extend google.protobuf.FieldOptions { + repeated string CopyTo = 82001; + repeated string AsMap = 82002; +} + diff --git a/ydb/core/config/protos/ya.make b/ydb/core/config/protos/ya.make new file mode 100644 index 000000000000..3ba2ab088188 --- /dev/null +++ b/ydb/core/config/protos/ya.make @@ -0,0 +1,9 @@ +PROTO_LIBRARY() + +SRCS( + marker.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/core/config/tools/protobuf_plugin/main.cpp b/ydb/core/config/tools/protobuf_plugin/main.cpp new file mode 100644 index 000000000000..4ffbd4e48ed9 --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/main.cpp @@ -0,0 +1,279 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +using TVariables = std::map; + +using namespace google::protobuf::compiler; +using namespace google::protobuf; +using namespace NKikimr::NProtobuf; + +class TPrinter { +public: + explicit TPrinter(OutputDirectory* output, const TString& fileName, const TString& scope) + : Output(output) + , FileName(fileName) + , Scope(scope) + { + } + + io::Printer* operator->() { + if (!Printer) { + Stream.Reset(Output->OpenForInsert(FileName, Scope)); + Printer.ConstructInPlace(Stream.Get(), '$'); + } + + return Printer.Get(); + } + +private: + OutputDirectory* Output; + const TString FileName; + const TString Scope; + + THolder Stream; + TMaybe Printer; + +}; // TPrinter + + +class TMessageGenerator { +public: + explicit TMessageGenerator(const Descriptor* message, OutputDirectory* output) + : Message(message) + , Header(output, HeaderFileName(message), ClassScope(message)) + , HeaderIncludes(output, HeaderFileName(message), IncludesScope()) + , Vars({ + {"class", ClassName(message)}, + {"fqMessageClass", FullyQualifiedClassName(Message)}, + }) + { + } + + void Generate() { + if (Message->options().GetExtension(NKikimrConfig::NMarkers::Root)) { + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + if (field->is_repeated()) { + continue; + } + if (auto* fieldMessage = field->message_type()) { + Vars["field"] = field->name(); + Vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); + Vars["fieldNumber"] = std::to_string(field->number()); + Header->Print(Vars, "/* BEGIN generated by config extension */\n"); + Header->Print(Vars, "struct T$field$FieldTag {};\n"); + Header->Print(Vars, "constexpr inline static std::tuple<\n"); + Header->Indent(); + Header->Print(Vars, "bool ($fqMessageClass$::*)() const,\n"); + Header->Print(Vars, "const $fqFieldClass$& ($fqMessageClass$::*)() const,\n"); + Header->Print(Vars, "$fqFieldClass$* ($fqMessageClass$::*)()\n"); + Header->Outdent(); + Header->Print(Vars, "> GetFieldAccessorsByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); + Header->Indent(); + Header->Print(Vars, "return std::tuple{\n"); + Header->Indent(); + Header->Print(Vars, "&$fqMessageClass$::Has$field$,\n"); + Header->Print(Vars, "&$fqMessageClass$::Get$field$,\n"); + Header->Print(Vars, "&$fqMessageClass$::Mutable$field$\n"); + Header->Outdent(); + Header->Print(Vars, "};\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "constexpr inline static ::NProtoBuf::uint32 GetFieldIdByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); + Header->Indent(); + Header->Print(Vars, "return $fieldNumber$;\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "/* END generated by config extension */\n"); + } + } + } + + + if (Message->options().GetExtension(NKikimrConfig::NMarkers::CombinedType)) { + TMap> outputs; + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + auto opts = field->options(); + for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::CopyTo); ++i) { + outputs[opts.GetExtension(NKikimrConfig::NMarkers::CopyTo, i)].insert(field); + } + } + + for (const auto& [output, fields] : outputs) { + Vars["output"] = output; + Header->Print(Vars, "/* BEGIN generated by config extension */\n"); + Header->Print(Vars, "template \n"); + Header->Print(Vars, "void CopyTo$output$(TOut& out) const {\n"); + Header->Indent(); + for (const auto* field : fields) { + Vars["field"] = field->name(); + + if (!field->is_repeated()) { + if (field->message_type()) { + Header->Print(Vars, "if (Has$field$()) {\n"); + Header->Indent(); + Header->Print(Vars, "out.Mutable$field$()->CopyFrom(Get$field$());\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + } else if (field->is_optional()) { + Header->Print(Vars, "if (Has$field$()) {\n"); + Header->Indent(); + Header->Print(Vars, "out.Set$field$(Get$field$());\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + } else { + Header->Print(Vars, "out.Set$field$(Get$field$());\n"); + } + } else { + if (field->message_type()) { + Header->Print(Vars, "for (size_t i = 0; i < $field$Size(); ++i) {\n"); + Header->Indent(); + Header->Print(Vars, "out.Add$field$()->CopyFrom(Get$field$(i));\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + } else { + Header->Print(Vars, "for (const auto& field : Get$field$()) {\n"); + Header->Indent(); + Header->Print(Vars, "out.Add$field$(field);\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + } + } + } + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "/* END generated by config extension */\n"); + } + } + + + if (Message->options().GetExtension(NKikimrConfig::NMarkers::WithMapType)) { + HeaderIncludes->Print(Vars, "#include \n"); + TMap> outputs; + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + auto opts = field->options(); + for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::AsMap); ++i) { + outputs[opts.GetExtension(NKikimrConfig::NMarkers::AsMap, i)].insert(field); + } + } + + // FIXME validate they have same type + + for (const auto& [output, fields] : outputs) { + if (auto* fieldMessage = (*fields.begin())->message_type()) { // FIXME for other classes + Vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); + } + Vars["output"] = output; + Header->Print(Vars, "/* BEGIN generated by config extension */\n"); + Header->Print(Vars, "size_t $output$Size(const TProtoStringType& str) const {\n"); + Header->Indent(); + Header->Print(Vars, "static std::map sizeHandlers{\n"); + Header->Indent(); + for (const auto* field : fields) { + Vars["field"] = field->name(); + Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::$field$Size},\n"); + } + Header->Outdent(); + Header->Print(Vars, "};\n"); + Header->Print(Vars, "auto it = sizeHandlers.find(str);\n"); + Header->Print(Vars, "return it == sizeHandlers.end() ? 0 : (this->*(it->second))();\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "$fqFieldClass$* Add$output$(const TProtoStringType& str) {\n"); + Header->Indent(); + Header->Print(Vars, "static std::map addHandlers{\n"); + Header->Indent(); + for (const auto* field : fields) { + Vars["field"] = field->name(); + Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Add$field$},\n"); + } + Header->Outdent(); + Header->Print(Vars, "};\n"); + Header->Print(Vars, "auto it = addHandlers.find(str);\n"); + Header->Print(Vars, "return it == addHandlers.end() ? nullptr : (this->*(it->second))();\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "const ::google::protobuf::RepeatedPtrField<$fqFieldClass$>& Get$output$(const TProtoStringType& str) const {\n"); + Header->Indent(); + Header->Print(Vars, "static std::map& ($fqMessageClass$::*)() const> getHandlers{\n"); + Header->Indent(); + for (const auto* field : fields) { + Vars["field"] = field->name(); + Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Get$field$},\n"); + } + Header->Outdent(); + Header->Print(Vars, "};\n"); + Header->Print(Vars, "return (this->*getHandlers.at(str))();\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "::google::protobuf::RepeatedPtrField<$fqFieldClass$>* Mutable$output$(const TProtoStringType& str) {\n"); + Header->Indent(); + Header->Print(Vars, "static std::map* ($fqMessageClass$::*)()> mutableHandlers{\n"); + Header->Indent(); + for (const auto* field : fields) { + Vars["field"] = field->name(); + Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Mutable$field$},\n"); + } + Header->Outdent(); + Header->Print(Vars, "};\n"); + Header->Print(Vars, "return (this->*mutableHandlers.at(str))();\n"); + Header->Outdent(); + Header->Print(Vars, "}\n"); + Header->Print(Vars, "/* END generated by config extension */\n"); + } + + } + } +private: + const Descriptor* Message; + TPrinter Header; + TPrinter HeaderIncludes; + TVariables Vars; +}; + +class TCodeGenerator: public CodeGenerator { + bool Generate( + const FileDescriptor* file, + const TProtoStringType&, + OutputDirectory* output, + TProtoStringType*) const override final + { + + for (auto i = 0; i < file->message_type_count(); ++i) { + TMessageGenerator mg(file->message_type(i), output); + mg.Generate(); + } + + return true; + } + + uint64_t GetSupportedFeatures() const override + { + return FEATURE_PROTO3_OPTIONAL; + } + +}; // TCodeGenerator + +int main(int argc, char* argv[]) { + TCodeGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/ydb/core/config/tools/protobuf_plugin/ut.cpp b/ydb/core/config/tools/protobuf_plugin/ut.cpp new file mode 100644 index 000000000000..19d4e843a3ab --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut.cpp @@ -0,0 +1,24 @@ +#include + +#include + +Y_UNIT_TEST_SUITE(ValidationTests) { + Y_UNIT_TEST(CanDispatchByTag) { + NKikimrConfig::ActualConfigMessage msg; + + msg.MutableField21(); + + auto [has1, get1, mut1] = NKikimrConfig::ActualConfigMessage::GetFieldAccessorsByFieldTag(NKikimrConfig::ActualConfigMessage::TField1FieldTag{}); + auto [has21, get21, mut21] = NKikimrConfig::ActualConfigMessage::GetFieldAccessorsByFieldTag(NKikimrConfig::ActualConfigMessage::TField21FieldTag{}); + + Y_UNUSED(get21, get1); + + UNIT_ASSERT(!(msg.*has1)()); + UNIT_ASSERT((msg.*has21)()); + + (msg.*mut1)(); + + UNIT_ASSERT(msg.HasField1()); + UNIT_ASSERT((msg.*has1)()); + } +} diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/config_root_test.proto b/ydb/core/config/tools/protobuf_plugin/ut/protos/config_root_test.proto new file mode 100644 index 000000000000..f1baf5e74101 --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/config_root_test.proto @@ -0,0 +1,19 @@ +import "ydb/core/config/protos/marker.proto"; + +package NKikimrConfig; +option java_package = "ru.yandex.kikimr.proto"; + +message AdditionalMessage { +} + +message AnotherMessage { + optional string Content = 1; + optional AdditionalMessage Field1 = 2; +} + +message ActualConfigMessage { + option (NMarkers.Root) = true; + optional AdditionalMessage Field1 = 1; + optional AdditionalMessage Field21 = 21; + optional string Field201 = 201; +} diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make new file mode 100644 index 000000000000..e3dd1f035e36 --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make @@ -0,0 +1,15 @@ +PROTO_LIBRARY() + +PEERDIR( + ydb/core/config/protos +) + +SRCS( + config_root_test.proto +) + +CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/core/config/tools/protobuf_plugin/ut/ya.make b/ydb/core/config/tools/protobuf_plugin/ut/ya.make new file mode 100644 index 000000000000..924b2d6a2f62 --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut/ya.make @@ -0,0 +1,18 @@ +UNITTEST_FOR(ydb/core/config/tools/protobuf_plugin) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + library/cpp/testing/unittest + ydb/core/config/tools/protobuf_plugin/ut/protos +) + +SRCS( + ut.cpp +) + +END() diff --git a/ydb/core/config/tools/protobuf_plugin/ya.make b/ydb/core/config/tools/protobuf_plugin/ya.make new file mode 100644 index 000000000000..91a255558b6f --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ya.make @@ -0,0 +1,17 @@ +PROGRAM(config_proto_plugin) + +PEERDIR( + contrib/libs/protoc + ydb/public/lib/protobuf + ydb/core/config/protos +) + +SRCS( + main.cpp +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/config/tools/ya.make b/ydb/core/config/tools/ya.make new file mode 100644 index 000000000000..61527259cbc6 --- /dev/null +++ b/ydb/core/config/tools/ya.make @@ -0,0 +1,3 @@ +RECURSE( + protobuf_plugin +) diff --git a/ydb/core/config/ya.make b/ydb/core/config/ya.make index 06efc6d2a17c..652ae9aba9f8 100644 --- a/ydb/core/config/ya.make +++ b/ydb/core/config/ya.make @@ -1,4 +1,6 @@ RECURSE( + protos + tools utils ) From bb94b4d99b54d68aa58d66eda7d8ed0f33be6bba Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 18:57:36 +0000 Subject: [PATCH 04/10] mark config root --- ydb/core/protos/config.proto | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 06751791a5a9..f618c1962b2d 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1,9 +1,12 @@ import "google/protobuf/descriptor.proto"; -import "ydb/library/actors/protos/interconnect.proto"; +import "ydb/core/config/proto_options/marker.proto"; +import "ydb/core/fq/libs/config/protos/fq_config.proto"; +import "ydb/core/protos/alloc.proto"; +import "ydb/core/protos/auth.proto"; import "ydb/core/protos/blobstorage.proto"; import "ydb/core/protos/blobstorage_config.proto"; -import "ydb/core/protos/blobstorage_vdisk_config.proto"; import "ydb/core/protos/blobstorage_pdisk_config.proto"; +import "ydb/core/protos/blobstorage_vdisk_config.proto"; import "ydb/core/protos/cms.proto"; import "ydb/core/protos/config_units.proto"; import "ydb/core/protos/counters_schemeshard.proto"; @@ -11,12 +14,14 @@ import "ydb/core/protos/datashard_config.proto"; import "ydb/core/protos/drivemodel.proto"; import "ydb/core/protos/feature_flags.proto"; import "ydb/core/protos/flat_scheme_op.proto"; -import "ydb/core/protos/http_config.proto"; import "ydb/core/protos/hive.proto"; +import "ydb/core/protos/http_config.proto"; +import "ydb/core/protos/key.proto"; import "ydb/core/protos/kqp.proto"; import "ydb/core/protos/local.proto"; import "ydb/core/protos/netclassifier.proto"; import "ydb/core/protos/node_broker.proto"; +import "ydb/core/protos/node_limits.proto"; import "ydb/core/protos/pqconfig.proto"; import "ydb/core/protos/resource_broker.proto"; import "ydb/core/protos/shared_cache.proto"; @@ -26,11 +31,7 @@ import "ydb/core/protos/tablet.proto"; import "ydb/core/protos/tablet_database.proto"; import "ydb/core/protos/tenant_pool.proto"; import "ydb/core/protos/tenant_slot_broker.proto"; -import "ydb/core/protos/auth.proto"; -import "ydb/core/protos/key.proto"; -import "ydb/core/protos/alloc.proto"; -import "ydb/core/protos/node_limits.proto"; -import "ydb/core/fq/libs/config/protos/fq_config.proto"; +import "ydb/library/actors/protos/interconnect.proto"; import "ydb/library/yql/providers/common/proto/gateways_config.proto"; package NKikimrConfig; @@ -1706,6 +1707,7 @@ message TLabel { } message TAppConfig { + option (NMarkers.Root) = true; optional TActorSystemConfig ActorSystemConfig = 1; optional TLogConfig LogConfig = 2; optional TStaticNameserviceConfig NameserviceConfig = 3; From ba3d3df59eaf2e90020c9b51ae7032ddf45bcb06 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 19:01:24 +0000 Subject: [PATCH 05/10] fix plugin usage --- ydb/core/protos/ya.make | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 13c7f54a784f..67d7f8a042df 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -145,6 +145,7 @@ GENERATE_ENUM_SERIALIZATION(datashard_load.pb.h) PEERDIR( ydb/library/actors/protos + ydb/core/config/protos ydb/core/fq/libs/config/protos ydb/core/scheme/protos ydb/library/login/protos @@ -160,6 +161,8 @@ PEERDIR( ydb/library/ydb_issue/proto ) +CPP_PROTO_PLUGIN0(config_proto_plugin ydb/config/tools/protobuf_plugin) + EXCLUDE_TAGS(GO_PROTO) END() From fd5d302940df6d9fd69e924c34af7e56d95597ea Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 19:03:23 +0000 Subject: [PATCH 06/10] fix plugin usage --- ydb/core/protos/config.proto | 2 +- ydb/core/protos/ya.make | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index f618c1962b2d..3f1daaef3ec0 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1,5 +1,5 @@ import "google/protobuf/descriptor.proto"; -import "ydb/core/config/proto_options/marker.proto"; +import "ydb/core/config/protos/marker.proto"; import "ydb/core/fq/libs/config/protos/fq_config.proto"; import "ydb/core/protos/alloc.proto"; import "ydb/core/protos/auth.proto"; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 67d7f8a042df..3198079dc9d4 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -161,7 +161,7 @@ PEERDIR( ydb/library/ydb_issue/proto ) -CPP_PROTO_PLUGIN0(config_proto_plugin ydb/config/tools/protobuf_plugin) +CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin) EXCLUDE_TAGS(GO_PROTO) From 286ad41adb754d07a7a4e1b05e140a64e11d4be8 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 8 Feb 2024 21:57:25 +0000 Subject: [PATCH 07/10] finish generator beautifying --- .../config/tools/protobuf_plugin/main.cpp | 372 +++++++++--------- .../lib/protobuf/base_message_generator.h | 32 ++ ydb/public/lib/protobuf/helpers.h | 45 +++ ydb/public/lib/protobuf/macro.h | 2 + ydb/public/lib/protobuf/scoped_file_printer.h | 42 ++ ydb/public/lib/protobuf/udnef_macro.h | 2 + ydb/public/lib/validation/main.cpp | 30 +- 7 files changed, 317 insertions(+), 208 deletions(-) create mode 100644 ydb/public/lib/protobuf/base_message_generator.h create mode 100644 ydb/public/lib/protobuf/macro.h create mode 100644 ydb/public/lib/protobuf/scoped_file_printer.h create mode 100644 ydb/public/lib/protobuf/udnef_macro.h diff --git a/ydb/core/config/tools/protobuf_plugin/main.cpp b/ydb/core/config/tools/protobuf_plugin/main.cpp index 4ffbd4e48ed9..32c111e6904f 100644 --- a/ydb/core/config/tools/protobuf_plugin/main.cpp +++ b/ydb/core/config/tools/protobuf_plugin/main.cpp @@ -16,238 +16,250 @@ #include #include +#include #include - -using TVariables = std::map; +#include using namespace google::protobuf::compiler; using namespace google::protobuf; using namespace NKikimr::NProtobuf; -class TPrinter { -public: - explicit TPrinter(OutputDirectory* output, const TString& fileName, const TString& scope) - : Output(output) - , FileName(fileName) - , Scope(scope) - { - } - - io::Printer* operator->() { - if (!Printer) { - Stream.Reset(Output->OpenForInsert(FileName, Scope)); - Printer.ConstructInPlace(Stream.Get(), '$'); - } - - return Printer.Get(); - } +constexpr TStringBuf PLUGIN_NAME = "config"; +class TMessageGenerator + : public TBaseMessageGenerator +{ private: - OutputDirectory* Output; - const TString FileName; - const TString Scope; - - THolder Stream; - TMaybe Printer; -}; // TPrinter + void GenerateConfigRoot(TVars vars) { + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + if (field->is_repeated()) { + continue; + } + if (auto* fieldMessage = field->message_type()) { + vars["field"] = field->name(); + vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); + vars["fieldNumber"] = std::to_string(field->number()); + WITH_PLUGIN_MARKUP(Header, PLUGIN_NAME) { + Header->Print(vars, "struct T$field$FieldTag {};\n"); -class TMessageGenerator { -public: - explicit TMessageGenerator(const Descriptor* message, OutputDirectory* output) - : Message(message) - , Header(output, HeaderFileName(message), ClassScope(message)) - , HeaderIncludes(output, HeaderFileName(message), IncludesScope()) - , Vars({ - {"class", ClassName(message)}, - {"fqMessageClass", FullyQualifiedClassName(Message)}, - }) - { - } + Header->Print(vars, "constexpr inline static std::tuple<\n"); + WITH_INDENT(Header) { + Header->Print(vars, + "bool ($fqMessageClass$::*)() const,\n" + "const $fqFieldClass$& ($fqMessageClass$::*)() const,\n" + "$fqFieldClass$* ($fqMessageClass$::*)()\n" + ); + } + Header->Print(vars, "> GetFieldAccessorsByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "return std::tuple{\n"); + WITH_INDENT(Header) { + Header->Print(vars, + "&$fqMessageClass$::Has$field$,\n" + "&$fqMessageClass$::Get$field$,\n" + "&$fqMessageClass$::Mutable$field$\n" + ); + } + Header->Print(vars, "};\n"); + } + Header->Print(vars, "}\n"); - void Generate() { - if (Message->options().GetExtension(NKikimrConfig::NMarkers::Root)) { - for (auto i = 0; i < Message->field_count(); ++i) { - const FieldDescriptor* field = Message->field(i); - if (field->is_repeated()) { - continue; - } - if (auto* fieldMessage = field->message_type()) { - Vars["field"] = field->name(); - Vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); - Vars["fieldNumber"] = std::to_string(field->number()); - Header->Print(Vars, "/* BEGIN generated by config extension */\n"); - Header->Print(Vars, "struct T$field$FieldTag {};\n"); - Header->Print(Vars, "constexpr inline static std::tuple<\n"); - Header->Indent(); - Header->Print(Vars, "bool ($fqMessageClass$::*)() const,\n"); - Header->Print(Vars, "const $fqFieldClass$& ($fqMessageClass$::*)() const,\n"); - Header->Print(Vars, "$fqFieldClass$* ($fqMessageClass$::*)()\n"); - Header->Outdent(); - Header->Print(Vars, "> GetFieldAccessorsByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); - Header->Indent(); - Header->Print(Vars, "return std::tuple{\n"); - Header->Indent(); - Header->Print(Vars, "&$fqMessageClass$::Has$field$,\n"); - Header->Print(Vars, "&$fqMessageClass$::Get$field$,\n"); - Header->Print(Vars, "&$fqMessageClass$::Mutable$field$\n"); - Header->Outdent(); - Header->Print(Vars, "};\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "constexpr inline static ::NProtoBuf::uint32 GetFieldIdByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); - Header->Indent(); - Header->Print(Vars, "return $fieldNumber$;\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "/* END generated by config extension */\n"); + Header->Print(vars, "constexpr inline static ::NProtoBuf::uint32 GetFieldIdByFieldTag($fqMessageClass$::T$field$FieldTag) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "return $fieldNumber$;\n"); + } + Header->Print(vars, "}\n"); } } } + } - - if (Message->options().GetExtension(NKikimrConfig::NMarkers::CombinedType)) { - TMap> outputs; - for (auto i = 0; i < Message->field_count(); ++i) { - const FieldDescriptor* field = Message->field(i); - auto opts = field->options(); - for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::CopyTo); ++i) { - outputs[opts.GetExtension(NKikimrConfig::NMarkers::CopyTo, i)].insert(field); - } + void GenerateCombinedType(TVars vars) { + TMap> outputs; + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + auto opts = field->options(); + for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::CopyTo); ++i) { + outputs[opts.GetExtension(NKikimrConfig::NMarkers::CopyTo, i)].insert(field); } + } - for (const auto& [output, fields] : outputs) { - Vars["output"] = output; - Header->Print(Vars, "/* BEGIN generated by config extension */\n"); - Header->Print(Vars, "template \n"); - Header->Print(Vars, "void CopyTo$output$(TOut& out) const {\n"); + for (const auto& [output, fields] : outputs) { + vars["output"] = output; + WITH_PLUGIN_MARKUP(Header, PLUGIN_NAME) { + Header->Print(vars, "template \n"); + Header->Print(vars, "void CopyTo$output$(TOut& out) const {\n"); Header->Indent(); for (const auto* field : fields) { - Vars["field"] = field->name(); + vars["field"] = field->name(); if (!field->is_repeated()) { if (field->message_type()) { - Header->Print(Vars, "if (Has$field$()) {\n"); - Header->Indent(); - Header->Print(Vars, "out.Mutable$field$()->CopyFrom(Get$field$());\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); + Header->Print(vars, "if (Has$field$()) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "out.Mutable$field$()->CopyFrom(Get$field$());\n"); + } + Header->Print(vars, "}\n"); } else if (field->is_optional()) { - Header->Print(Vars, "if (Has$field$()) {\n"); - Header->Indent(); - Header->Print(Vars, "out.Set$field$(Get$field$());\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); + Header->Print(vars, "if (Has$field$()) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "out.Set$field$(Get$field$());\n"); + } + Header->Print(vars, "}\n"); } else { - Header->Print(Vars, "out.Set$field$(Get$field$());\n"); + Header->Print(vars, "out.Set$field$(Get$field$());\n"); } } else { if (field->message_type()) { - Header->Print(Vars, "for (size_t i = 0; i < $field$Size(); ++i) {\n"); - Header->Indent(); - Header->Print(Vars, "out.Add$field$()->CopyFrom(Get$field$(i));\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); + Header->Print(vars, "for (size_t i = 0; i < $field$Size(); ++i) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "out.Add$field$()->CopyFrom(Get$field$(i));\n"); + } + Header->Print(vars, "}\n"); } else { - Header->Print(Vars, "for (const auto& field : Get$field$()) {\n"); - Header->Indent(); - Header->Print(Vars, "out.Add$field$(field);\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); + Header->Print(vars, "for (const auto& field : Get$field$()) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "out.Add$field$(field);\n"); + } + Header->Print(vars, "}\n"); } } } Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "/* END generated by config extension */\n"); + Header->Print(vars, "}\n"); } } + } - - if (Message->options().GetExtension(NKikimrConfig::NMarkers::WithMapType)) { - HeaderIncludes->Print(Vars, "#include \n"); - TMap> outputs; - for (auto i = 0; i < Message->field_count(); ++i) { - const FieldDescriptor* field = Message->field(i); - auto opts = field->options(); - for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::AsMap); ++i) { - outputs[opts.GetExtension(NKikimrConfig::NMarkers::AsMap, i)].insert(field); + void GenerateSizeFields(TVars vars, const TSet& fields) { + Header->Print(vars, "size_t $output$Size(const TProtoStringType& str) const {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "static std::map sizeHandlers{\n"); + WITH_INDENT(Header) { + for (const auto* field : fields) { + vars["field"] = field->name(); + Header->Print(vars, "{\"$field$\", &$fqMessageClass$::$field$Size},\n"); } } + Header->Print(vars, "};\n"); - // FIXME validate they have same type + Header->Print(vars, "auto it = sizeHandlers.find(str);\n"); + Header->Print(vars, "return it == sizeHandlers.end() ? 0 : (this->*(it->second))();\n"); + } + Header->Print(vars, "}\n"); + } - for (const auto& [output, fields] : outputs) { - if (auto* fieldMessage = (*fields.begin())->message_type()) { // FIXME for other classes - Vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); - } - Vars["output"] = output; - Header->Print(Vars, "/* BEGIN generated by config extension */\n"); - Header->Print(Vars, "size_t $output$Size(const TProtoStringType& str) const {\n"); - Header->Indent(); - Header->Print(Vars, "static std::map sizeHandlers{\n"); - Header->Indent(); + void GenerateAddFields(TVars vars, const TSet& fields) { + Header->Print(vars, "$fqFieldClass$* Add$output$(const TProtoStringType& str) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "static std::map addHandlers{\n"); + WITH_INDENT(Header) { for (const auto* field : fields) { - Vars["field"] = field->name(); - Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::$field$Size},\n"); + vars["field"] = field->name(); + Header->Print(vars, "{\"$field$\", &$fqMessageClass$::Add$field$},\n"); } - Header->Outdent(); - Header->Print(Vars, "};\n"); - Header->Print(Vars, "auto it = sizeHandlers.find(str);\n"); - Header->Print(Vars, "return it == sizeHandlers.end() ? 0 : (this->*(it->second))();\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "$fqFieldClass$* Add$output$(const TProtoStringType& str) {\n"); - Header->Indent(); - Header->Print(Vars, "static std::map addHandlers{\n"); - Header->Indent(); + } + Header->Print(vars, "};\n"); + + Header->Print(vars, "auto it = addHandlers.find(str);\n"); + Header->Print(vars, "return it == addHandlers.end() ? nullptr : (this->*(it->second))();\n"); + } + Header->Print(vars, "}\n"); + } + + void GenerateGetFields(TVars vars, const TSet& fields) { + Header->Print(vars, "const ::google::protobuf::RepeatedPtrField<$fqFieldClass$>& Get$output$(const TProtoStringType& str) const {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "static std::map& ($fqMessageClass$::*)() const> getHandlers{\n"); + WITH_INDENT(Header) { for (const auto* field : fields) { - Vars["field"] = field->name(); - Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Add$field$},\n"); + vars["field"] = field->name(); + Header->Print(vars, "{\"$field$\", &$fqMessageClass$::Get$field$},\n"); } - Header->Outdent(); - Header->Print(Vars, "};\n"); - Header->Print(Vars, "auto it = addHandlers.find(str);\n"); - Header->Print(Vars, "return it == addHandlers.end() ? nullptr : (this->*(it->second))();\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "const ::google::protobuf::RepeatedPtrField<$fqFieldClass$>& Get$output$(const TProtoStringType& str) const {\n"); - Header->Indent(); - Header->Print(Vars, "static std::map& ($fqMessageClass$::*)() const> getHandlers{\n"); - Header->Indent(); + } + Header->Print(vars, "};\n"); + + Header->Print(vars, "return (this->*getHandlers.at(str))();\n"); + } + Header->Print(vars, "}\n"); + } + + void GenerateMutableFields(TVars vars, const TSet& fields) { + Header->Print(vars, "::google::protobuf::RepeatedPtrField<$fqFieldClass$>* Mutable$output$(const TProtoStringType& str) {\n"); + WITH_INDENT(Header) { + Header->Print(vars, "static std::map* ($fqMessageClass$::*)()> mutableHandlers{\n"); + WITH_INDENT(Header) { for (const auto* field : fields) { - Vars["field"] = field->name(); - Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Get$field$},\n"); + vars["field"] = field->name(); + Header->Print(vars, "{\"$field$\", &$fqMessageClass$::Mutable$field$},\n"); } - Header->Outdent(); - Header->Print(Vars, "};\n"); - Header->Print(Vars, "return (this->*getHandlers.at(str))();\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "::google::protobuf::RepeatedPtrField<$fqFieldClass$>* Mutable$output$(const TProtoStringType& str) {\n"); - Header->Indent(); - Header->Print(Vars, "static std::map* ($fqMessageClass$::*)()> mutableHandlers{\n"); - Header->Indent(); + } + Header->Print(vars, "};\n"); + + Header->Print(vars, "return (this->*mutableHandlers.at(str))();\n"); + } + Header->Print(vars, "}\n"); + } + + void GenerateFieldsMap(TVars vars) { + TMap> outputs; + for (auto i = 0; i < Message->field_count(); ++i) { + const FieldDescriptor* field = Message->field(i); + auto opts = field->options(); + for (int i = 0; i < opts.ExtensionSize(NKikimrConfig::NMarkers::AsMap); ++i) { + outputs[opts.GetExtension(NKikimrConfig::NMarkers::AsMap, i)].insert(field); + } + } + + if (!outputs) { + return; + } + + WITH_PLUGIN_MARKUP(HeaderIncludes, PLUGIN_NAME) { + HeaderIncludes->Print(Vars, "#include \n"); + } + + for (const auto& [output, fields] : outputs) { + if (auto* fieldMessage = (*fields.begin())->message_type()) { // TODO implement for other classes for (const auto* field : fields) { - Vars["field"] = field->name(); - Header->Print(Vars, "{\"$field$\", &$fqMessageClass$::Mutable$field$},\n"); + Y_ABORT_UNLESS(fieldMessage->full_name() == field->full_name(), "Messages in map MUST have the same type"); } - Header->Outdent(); - Header->Print(Vars, "};\n"); - Header->Print(Vars, "return (this->*mutableHandlers.at(str))();\n"); - Header->Outdent(); - Header->Print(Vars, "}\n"); - Header->Print(Vars, "/* END generated by config extension */\n"); + vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); + } else { + Y_ABORT("Types other thans Message is not supported yet."); } + vars["output"] = output; + + WITH_PLUGIN_MARKUP(Header, PLUGIN_NAME) { + GenerateSizeFields(vars, fields); + GenerateAddFields(vars, fields); + GenerateGetFields(vars, fields); + GenerateMutableFields(vars, fields); + } + } + } + +public: + + using TBaseMessageGenerator::TBaseMessageGenerator; + + void Generate() { + if (Message->options().GetExtension(NKikimrConfig::NMarkers::Root)) { + GenerateConfigRoot(Vars); + } + + if (Message->options().GetExtension(NKikimrConfig::NMarkers::CombinedType)) { + GenerateCombinedType(Vars); + } + + if (Message->options().GetExtension(NKikimrConfig::NMarkers::WithMapType)) { + GenerateFieldsMap(Vars); } } -private: - const Descriptor* Message; - TPrinter Header; - TPrinter HeaderIncludes; - TVariables Vars; }; class TCodeGenerator: public CodeGenerator { diff --git a/ydb/public/lib/protobuf/base_message_generator.h b/ydb/public/lib/protobuf/base_message_generator.h new file mode 100644 index 000000000000..6fe31797424a --- /dev/null +++ b/ydb/public/lib/protobuf/base_message_generator.h @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +#include + +namespace NKikimr::NProtobuf { + +class TBaseMessageGenerator { +public: + using TVars = std::map; + + explicit TBaseMessageGenerator(const google::protobuf::Descriptor* message, google::protobuf::compiler::OutputDirectory* output) + : Message(message) + , Header(output, HeaderFileName(message), ClassScope(message)) + , HeaderIncludes(output, HeaderFileName(message), IncludesScope()) + , Vars({ + {"class", ClassName(message)}, + {"fqMessageClass", FullyQualifiedClassName(Message)}, + }) + {} + +protected: + const google::protobuf::Descriptor* Message; + TScopedFilePrinter Header; + TScopedFilePrinter HeaderIncludes; + TVars Vars; +}; + +} // namespace NKikimr::NProtobuf diff --git a/ydb/public/lib/protobuf/helpers.h b/ydb/public/lib/protobuf/helpers.h index a598b0fa3a45..8a187d0ff7a1 100644 --- a/ydb/public/lib/protobuf/helpers.h +++ b/ydb/public/lib/protobuf/helpers.h @@ -1,8 +1,10 @@ #pragma once #include +#include #include +#include namespace NKikimr::NProtobuf { @@ -16,4 +18,47 @@ TString ClassName(const google::protobuf::Descriptor* message); bool IsCustomMessage(const google::protobuf::Descriptor* message); +struct IPrinter { + virtual ~IPrinter() = default; + virtual google::protobuf::io::Printer* operator->() = 0; +}; + +class TScopeIndentPrinterGuard + : public TNonCopyable +{ +public: + TScopeIndentPrinterGuard(IPrinter& printer) + : Printer(printer) + { + Printer->Indent(); + } + + ~TScopeIndentPrinterGuard() { + Printer->Outdent(); + } + +private: + IPrinter& Printer; +}; + +class TPluginMarkupPrinterGuard + : public TNonCopyable +{ +public: + TPluginMarkupPrinterGuard(IPrinter& printer, TStringBuf pluginName) + : Printer(printer) + , PluginName(pluginName) + { + Printer->Print({{"pluginName", PluginName}}, "/* BEGIN generated by $pluginName$ extension */\n"); + } + + ~TPluginMarkupPrinterGuard() { + Printer->Print({{"pluginName", PluginName}}, "/* END generated by $pluginName$ extension */\n"); + } + +private: + IPrinter& Printer; + TString PluginName; +}; + } // namespace NKikimr::NProtobuf diff --git a/ydb/public/lib/protobuf/macro.h b/ydb/public/lib/protobuf/macro.h new file mode 100644 index 000000000000..185fef2d546f --- /dev/null +++ b/ydb/public/lib/protobuf/macro.h @@ -0,0 +1,2 @@ +#define WITH_INDENT(printer) if (TScopeIndentPrinterGuard indentGuard(printer); true) +#define WITH_PLUGIN_MARKUP(printer, pluginName) if (TPluginMarkupPrinterGuard indentGuard(printer, pluginName); true) diff --git a/ydb/public/lib/protobuf/scoped_file_printer.h b/ydb/public/lib/protobuf/scoped_file_printer.h new file mode 100644 index 000000000000..f802aead6bd4 --- /dev/null +++ b/ydb/public/lib/protobuf/scoped_file_printer.h @@ -0,0 +1,42 @@ +#include "helpers.h" + +#include +#include +#include + +#include +#include +#include + +namespace NKikimr::NProtobuf { + +class TScopedFilePrinter + : public IPrinter +{ +public: + explicit TScopedFilePrinter(google::protobuf::compiler::OutputDirectory* output, const TString& fileName, const TString& scope) + : Output(output) + , FileName(fileName) + , Scope(scope) + { + } + + google::protobuf::io::Printer* operator->() override { + if (!Printer) { + Stream.Reset(Output->OpenForInsert(FileName, Scope)); + Printer.ConstructInPlace(Stream.Get(), '$'); + } + + return Printer.Get(); + } + +protected: + google::protobuf::compiler::OutputDirectory* Output; + const TString FileName; + const TString Scope; + + THolder Stream; + TMaybe Printer; +}; // TScopedFilePrinter + +} // namespace NKikimr::NProtobuf diff --git a/ydb/public/lib/protobuf/udnef_macro.h b/ydb/public/lib/protobuf/udnef_macro.h new file mode 100644 index 000000000000..81eb0f4d28bb --- /dev/null +++ b/ydb/public/lib/protobuf/udnef_macro.h @@ -0,0 +1,2 @@ +#undef WITH_INDENT +#undef WITH_PLUGIN_MARKUP diff --git a/ydb/public/lib/validation/main.cpp b/ydb/public/lib/validation/main.cpp index 3fb3be26c504..090322ac57a3 100644 --- a/ydb/public/lib/validation/main.cpp +++ b/ydb/public/lib/validation/main.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -22,34 +23,7 @@ using namespace google::protobuf; using namespace NKikimr::NProtobuf; using TVariables = std::map; - -class TPrinter { -public: - explicit TPrinter(OutputDirectory* output, const TString& fileName, const TString& scope) - : Output(output) - , FileName(fileName) - , Scope(scope) - { - } - - io::Printer* operator->() { - if (!Printer) { - Stream.Reset(Output->OpenForInsert(FileName, Scope)); - Printer.ConstructInPlace(Stream.Get(), '$'); - } - - return Printer.Get(); - } - -private: - OutputDirectory* Output; - const TString FileName; - const TString Scope; - - THolder Stream; - TMaybe Printer; - -}; // TPrinter +using TPrinter = NKikimr::NProtobuf::TScopedFilePrinter; bool IsScalarType(const FieldDescriptor* field) { switch (field->cpp_type()) { From 79433133ca503bb71ca36676eda16092d39f2b37 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 9 Feb 2024 11:57:41 +0000 Subject: [PATCH 08/10] add some tests --- ydb/core/config/tools/protobuf_plugin/ut.cpp | 151 ++++++++++++++++++ .../ut/protos/copy_to_test.proto | 54 +++++++ .../tools/protobuf_plugin/ut/protos/ya.make | 1 + ydb/public/lib/protobuf/macro.h | 4 +- 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 ydb/core/config/tools/protobuf_plugin/ut/protos/copy_to_test.proto diff --git a/ydb/core/config/tools/protobuf_plugin/ut.cpp b/ydb/core/config/tools/protobuf_plugin/ut.cpp index 19d4e843a3ab..65bf4e4c97e2 100644 --- a/ydb/core/config/tools/protobuf_plugin/ut.cpp +++ b/ydb/core/config/tools/protobuf_plugin/ut.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -21,4 +22,154 @@ Y_UNIT_TEST_SUITE(ValidationTests) { UNIT_ASSERT(msg.HasField1()); UNIT_ASSERT((msg.*has1)()); } + + Y_UNIT_TEST(CanCopyTo) { + NKikimrConfig::SourceMessage source; + NKikimrConfig::FirstSinkMessage firstSink; + NKikimrConfig::SecondSinkMessage secondSink; + + source.CopyToFirstSinkMessage(firstSink); + source.CopyToSecondSinkMessage(secondSink); + + UNIT_ASSERT(!source.HasStringField1()); + UNIT_ASSERT(!source.HasStringField2()); + UNIT_ASSERT(!source.HasStringField3()); + UNIT_ASSERT(!source.HasStringField4()); + UNIT_ASSERT(!source.StringField5Size()); + UNIT_ASSERT(!source.StringField6Size()); + UNIT_ASSERT(!source.StringField7Size()); + UNIT_ASSERT(!source.StringField8Size()); + UNIT_ASSERT(!source.HasComplexMessage1()); + UNIT_ASSERT(!source.HasComplexMessage2()); + UNIT_ASSERT(!source.HasComplexMessage3()); + UNIT_ASSERT(!source.HasComplexMessage4()); + UNIT_ASSERT(!source.ComplexMessage5Size()); + UNIT_ASSERT(!source.ComplexMessage6Size()); + UNIT_ASSERT(!source.ComplexMessage7Size()); + UNIT_ASSERT(!source.ComplexMessage8Size()); + + UNIT_ASSERT(!firstSink.HasStringField2()); + UNIT_ASSERT(!firstSink.HasStringField4()); + UNIT_ASSERT(!firstSink.StringField6Size()); + UNIT_ASSERT(!firstSink.StringField8Size()); + UNIT_ASSERT(!firstSink.HasComplexMessage2()); + UNIT_ASSERT(!firstSink.HasComplexMessage4()); + UNIT_ASSERT(!firstSink.ComplexMessage6Size()); + UNIT_ASSERT(!firstSink.ComplexMessage8Size()); + + UNIT_ASSERT(!secondSink.HasStringField3()); + UNIT_ASSERT(!secondSink.HasStringField4()); + UNIT_ASSERT(!secondSink.StringField7Size()); + UNIT_ASSERT(!secondSink.StringField8Size()); + UNIT_ASSERT(!secondSink.HasComplexMessage3()); + UNIT_ASSERT(!secondSink.HasComplexMessage4()); + UNIT_ASSERT(!secondSink.ComplexMessage7Size()); + UNIT_ASSERT(!secondSink.ComplexMessage8Size()); + + source.SetStringField1("string1"); + source.SetStringField2("string2"); + source.SetStringField3("string3"); + source.SetStringField4("string4"); + source.AddStringField5("string5-1"); + source.AddStringField5("string5-2"); + source.AddStringField6("string6-1"); + source.AddStringField6("string6-2"); + source.AddStringField6("string6-3"); + source.AddStringField7("string7-1"); + source.AddStringField7("string7-2"); + source.AddStringField7("string7-3"); + source.AddStringField7("string7-4"); + source.AddStringField8("string8-1"); + source.AddStringField8("string8-2"); + source.AddStringField8("string8-3"); + source.AddStringField8("string8-4"); + source.AddStringField8("string8-5"); + + source.SetIntField(1); + + source.MutableComplexMessage1()->SetStringField("cm1-string"); + source.MutableComplexMessage2()->SetStringField("cm2-string"); + source.MutableComplexMessage3()->SetStringField("cm3-string"); + source.MutableComplexMessage4()->SetStringField("cm4-string"); + source.AddComplexMessage5()->SetStringField("cm5-string-1"); + source.AddComplexMessage5()->SetStringField("cm5-string-2"); + source.AddComplexMessage6()->SetStringField("cm6-string-1"); + source.AddComplexMessage6()->SetStringField("cm6-string-2"); + source.AddComplexMessage6()->SetStringField("cm6-string-3"); + source.AddComplexMessage7()->SetStringField("cm7-string-1"); + source.AddComplexMessage7()->SetStringField("cm7-string-2"); + source.AddComplexMessage7()->SetStringField("cm7-string-3"); + source.AddComplexMessage7()->SetStringField("cm7-string-4"); + source.AddComplexMessage8()->SetStringField("cm8-string-1"); + source.AddComplexMessage8()->SetStringField("cm8-string-2"); + source.AddComplexMessage8()->SetStringField("cm8-string-3"); + source.AddComplexMessage8()->SetStringField("cm8-string-4"); + source.AddComplexMessage8()->SetStringField("cm8-string-5"); + + firstSink.SetIntField(2); + secondSink.SetIntField(3); + + TString expectedSource = source.ShortDebugString(); + + source.CopyToFirstSinkMessage(firstSink); + + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField2(), source.GetStringField2()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField4(), source.GetStringField4()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.StringField6Size(), source.StringField6Size()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField6(0), source.GetStringField6(0)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField6(1), source.GetStringField6(1)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField6(2), source.GetStringField6(2)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.StringField8Size(), source.StringField8Size()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField8(0), source.GetStringField8(0)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField8(1), source.GetStringField8(1)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField8(2), source.GetStringField8(2)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField8(3), source.GetStringField8(3)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetStringField8(4), source.GetStringField8(4)); + UNIT_ASSERT_VALUES_EQUAL(firstSink.ComplexMessage6Size(), source.ComplexMessage6Size()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage6(0).ShortDebugString(), source.GetComplexMessage6(0).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage6(1).ShortDebugString(), source.GetComplexMessage6(1).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage6(2).ShortDebugString(), source.GetComplexMessage6(2).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.ComplexMessage8Size(), source.ComplexMessage8Size()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage8(0).ShortDebugString(), source.GetComplexMessage8(0).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage8(1).ShortDebugString(), source.GetComplexMessage8(1).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage8(2).ShortDebugString(), source.GetComplexMessage8(2).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage8(3).ShortDebugString(), source.GetComplexMessage8(3).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetComplexMessage8(4).ShortDebugString(), source.GetComplexMessage8(4).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(firstSink.GetIntField(), 2); + + UNIT_ASSERT_VALUES_EQUAL(expectedSource, source.ShortDebugString()); + + TString expectedFirstSink = firstSink.ShortDebugString(); + + source.CopyToSecondSinkMessage(secondSink); + + UNIT_ASSERT_VALUES_EQUAL(expectedSource, source.ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(expectedFirstSink, firstSink.ShortDebugString()); + + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField3(), source.GetStringField3()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField4(), source.GetStringField4()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.StringField7Size(), source.StringField7Size()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField7(0), source.GetStringField7(0)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField7(1), source.GetStringField7(1)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField7(2), source.GetStringField7(2)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField7(3), source.GetStringField7(3)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.StringField8Size(), source.StringField8Size()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField8(0), source.GetStringField8(0)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField8(1), source.GetStringField8(1)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField8(2), source.GetStringField8(2)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField8(3), source.GetStringField8(3)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetStringField8(4), source.GetStringField8(4)); + UNIT_ASSERT_VALUES_EQUAL(secondSink.ComplexMessage7Size(), source.ComplexMessage7Size()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage7(0).ShortDebugString(), source.GetComplexMessage7(0).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage7(1).ShortDebugString(), source.GetComplexMessage7(1).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage7(2).ShortDebugString(), source.GetComplexMessage7(2).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage7(3).ShortDebugString(), source.GetComplexMessage7(3).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.ComplexMessage8Size(), source.ComplexMessage8Size()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(0).ShortDebugString(), source.GetComplexMessage8(0).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(1).ShortDebugString(), source.GetComplexMessage8(1).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(2).ShortDebugString(), source.GetComplexMessage8(2).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(3).ShortDebugString(), source.GetComplexMessage8(3).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(4).ShortDebugString(), source.GetComplexMessage8(4).ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL(secondSink.GetIntField(), 3); + } } diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/copy_to_test.proto b/ydb/core/config/tools/protobuf_plugin/ut/protos/copy_to_test.proto new file mode 100644 index 000000000000..29dc2bf22c6d --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/copy_to_test.proto @@ -0,0 +1,54 @@ +import "ydb/core/config/protos/marker.proto"; + +package NKikimrConfig; +option java_package = "ru.yandex.kikimr.proto"; + +message SomeComplexMessage { + optional string StringField = 1; + optional uint32 IntField = 2; +} + +message SourceMessage { + option (NMarkers.CombinedType) = true; + optional string StringField1 = 1; + optional string StringField2 = 2 [(NMarkers.CopyTo) = "FirstSinkMessage"]; + optional string StringField3 = 3 [(NMarkers.CopyTo) = "SecondSinkMessage"]; + optional string StringField4 = 4 [(NMarkers.CopyTo) = "FirstSinkMessage", (NMarkers.CopyTo) = "SecondSinkMessage"]; + repeated string StringField5 = 5; + repeated string StringField6 = 6 [(NMarkers.CopyTo) = "FirstSinkMessage"]; + repeated string StringField7 = 7 [(NMarkers.CopyTo) = "SecondSinkMessage"]; + repeated string StringField8 = 8 [(NMarkers.CopyTo) = "FirstSinkMessage", (NMarkers.CopyTo) = "SecondSinkMessage"]; + optional uint32 IntField = 9; + optional SomeComplexMessage ComplexMessage1 = 10; + optional SomeComplexMessage ComplexMessage2 = 11 [(NMarkers.CopyTo) = "FirstSinkMessage"]; + optional SomeComplexMessage ComplexMessage3 = 12 [(NMarkers.CopyTo) = "SecondSinkMessage"]; + optional SomeComplexMessage ComplexMessage4 = 13 [(NMarkers.CopyTo) = "FirstSinkMessage", (NMarkers.CopyTo) = "SecondSinkMessage"]; + repeated SomeComplexMessage ComplexMessage5 = 14; + repeated SomeComplexMessage ComplexMessage6 = 15 [(NMarkers.CopyTo) = "FirstSinkMessage"]; + repeated SomeComplexMessage ComplexMessage7 = 16 [(NMarkers.CopyTo) = "SecondSinkMessage"]; + repeated SomeComplexMessage ComplexMessage8 = 17 [(NMarkers.CopyTo) = "FirstSinkMessage", (NMarkers.CopyTo) = "SecondSinkMessage"]; +} + +message FirstSinkMessage { + optional string StringField2 = 1; + optional string StringField4 = 3; + repeated string StringField6 = 4; + repeated string StringField8 = 8; + optional uint32 IntField = 9; + optional SomeComplexMessage ComplexMessage2 = 101; + optional SomeComplexMessage ComplexMessage4 = 103; + repeated SomeComplexMessage ComplexMessage6 = 104; + repeated SomeComplexMessage ComplexMessage8 = 106; +} + +message SecondSinkMessage { + optional string StringField3 = 2; + optional string StringField4 = 3; + repeated string StringField7 = 5; + repeated string StringField8 = 8; + optional uint32 IntField = 9; + optional SomeComplexMessage ComplexMessage3 = 102; + optional SomeComplexMessage ComplexMessage4 = 103; + repeated SomeComplexMessage ComplexMessage7 = 105; + repeated SomeComplexMessage ComplexMessage8 = 106; +} diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make index e3dd1f035e36..4445e0207a4b 100644 --- a/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make @@ -6,6 +6,7 @@ PEERDIR( SRCS( config_root_test.proto + copy_to_test.proto ) CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin) diff --git a/ydb/public/lib/protobuf/macro.h b/ydb/public/lib/protobuf/macro.h index 185fef2d546f..8f947d2cc96b 100644 --- a/ydb/public/lib/protobuf/macro.h +++ b/ydb/public/lib/protobuf/macro.h @@ -1,2 +1,2 @@ -#define WITH_INDENT(printer) if (TScopeIndentPrinterGuard indentGuard(printer); true) -#define WITH_PLUGIN_MARKUP(printer, pluginName) if (TPluginMarkupPrinterGuard indentGuard(printer, pluginName); true) +#define WITH_INDENT(printer) if (::NKikimr::NProtobuf::TScopeIndentPrinterGuard indentGuard(printer); true) +#define WITH_PLUGIN_MARKUP(printer, pluginName) if (::NKikimr::NProtobuf::TPluginMarkupPrinterGuard indentGuard(printer, pluginName); true) From afa9d26617992e223199cef6bcbd93214fd2e185 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 9 Feb 2024 13:25:55 +0000 Subject: [PATCH 09/10] add more tests --- .../config/tools/protobuf_plugin/main.cpp | 15 +++- ydb/core/config/tools/protobuf_plugin/ut.cpp | 80 +++++++++++++++++++ .../ut/protos/as_map_test.proto | 23 ++++++ .../tools/protobuf_plugin/ut/protos/ya.make | 1 + 4 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 ydb/core/config/tools/protobuf_plugin/ut/protos/as_map_test.proto diff --git a/ydb/core/config/tools/protobuf_plugin/main.cpp b/ydb/core/config/tools/protobuf_plugin/main.cpp index 32c111e6904f..b38ac3904f5c 100644 --- a/ydb/core/config/tools/protobuf_plugin/main.cpp +++ b/ydb/core/config/tools/protobuf_plugin/main.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -164,8 +165,7 @@ class TMessageGenerator } Header->Print(vars, "};\n"); - Header->Print(vars, "auto it = addHandlers.find(str);\n"); - Header->Print(vars, "return it == addHandlers.end() ? nullptr : (this->*(it->second))();\n"); + Header->Print(vars, "return (this->*addHandlers.at(str))();\n"); } Header->Print(vars, "}\n"); } @@ -225,11 +225,18 @@ class TMessageGenerator for (const auto& [output, fields] : outputs) { if (auto* fieldMessage = (*fields.begin())->message_type()) { // TODO implement for other classes for (const auto* field : fields) { - Y_ABORT_UNLESS(fieldMessage->full_name() == field->full_name(), "Messages in map MUST have the same type"); + if (fieldMessage->full_name() != field->message_type()->full_name()) { + Cerr << "Messages in map MUST have the same type: " + << fieldMessage->full_name().c_str() << " != " + << field->message_type()->full_name().c_str() << Endl; + Y_ABORT("Invariant failed"); + } + + Y_ABORT_UNLESS(field->is_repeated(), "Only repeated fields are supported"); } vars["fqFieldClass"] = FullyQualifiedClassName(fieldMessage); } else { - Y_ABORT("Types other thans Message is not supported yet."); + Y_ABORT("Types other than Message is not supported yet."); } vars["output"] = output; diff --git a/ydb/core/config/tools/protobuf_plugin/ut.cpp b/ydb/core/config/tools/protobuf_plugin/ut.cpp index 65bf4e4c97e2..279ea2bf2be1 100644 --- a/ydb/core/config/tools/protobuf_plugin/ut.cpp +++ b/ydb/core/config/tools/protobuf_plugin/ut.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -172,4 +173,83 @@ Y_UNIT_TEST_SUITE(ValidationTests) { UNIT_ASSERT_VALUES_EQUAL(secondSink.GetComplexMessage8(4).ShortDebugString(), source.GetComplexMessage8(4).ShortDebugString()); UNIT_ASSERT_VALUES_EQUAL(secondSink.GetIntField(), 3); } + + Y_UNIT_TEST(MapType) { + NKikimrConfig::MessageWithMap msg; + TString nameBase = "Entry"; + for (int i = 1; i <= 10; ++i) { + TString name = nameBase + ToString(i); + UNIT_ASSERT_VALUES_EQUAL(msg.GetSomeMap(name).size(), 0); + } + + UNIT_CHECK_GENERATED_EXCEPTION(msg.GetSomeMap("otherName"), std::out_of_range); + UNIT_CHECK_GENERATED_EXCEPTION(msg.GetSomeMap("Entry0"), std::out_of_range); + + UNIT_ASSERT(!msg.Entry1Size()); + UNIT_ASSERT(!msg.Entry2Size()); + UNIT_ASSERT(!msg.Entry3Size()); + UNIT_ASSERT(!msg.Entry4Size()); + UNIT_ASSERT(!msg.Entry5Size()); + UNIT_ASSERT(!msg.Entry6Size()); + UNIT_ASSERT(!msg.Entry7Size()); + UNIT_ASSERT(!msg.Entry8Size()); + UNIT_ASSERT(!msg.Entry9Size()); + UNIT_ASSERT(!msg.Entry10Size()); + + for (int i = 1; i <= 10; ++i) { + TString name = nameBase + ToString(i); + msg.AddSomeMap(name)->SetStr(ToString(i)); + } + + UNIT_CHECK_GENERATED_EXCEPTION(msg.AddSomeMap("otherName"), std::out_of_range); + + UNIT_ASSERT_VALUES_EQUAL(msg.Entry1Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry1(0).GetStr(), "1"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry2Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry2(0).GetStr(), "2"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry3Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry3(0).GetStr(), "3"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry4Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry4(0).GetStr(), "4"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry5Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry5(0).GetStr(), "5"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry6Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry6(0).GetStr(), "6"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry7Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry7(0).GetStr(), "7"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry8Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry8(0).GetStr(), "8"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry9Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry9(0).GetStr(), "9"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry10Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry10(0).GetStr(), "10"); + + for (int i = 1; i <= 10; ++i) { + TString name = nameBase + ToString(i); + msg.MutableSomeMap(name)->Add()->SetStr(ToString(i * i)); + } + + UNIT_CHECK_GENERATED_EXCEPTION(msg.MutableSomeMap("otherName"), std::out_of_range); + + UNIT_ASSERT_VALUES_EQUAL(msg.Entry1Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry1(1).GetStr(), "1"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry2Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry2(1).GetStr(), "4"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry3Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry3(1).GetStr(), "9"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry4Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry4(1).GetStr(), "16"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry5Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry5(1).GetStr(), "25"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry6Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry6(1).GetStr(), "36"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry7Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry7(1).GetStr(), "49"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry8Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry8(1).GetStr(), "64"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry9Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry9(1).GetStr(), "81"); + UNIT_ASSERT_VALUES_EQUAL(msg.Entry10Size(), 2); + UNIT_ASSERT_VALUES_EQUAL(msg.GetEntry10(1).GetStr(), "100"); + } } diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/as_map_test.proto b/ydb/core/config/tools/protobuf_plugin/ut/protos/as_map_test.proto new file mode 100644 index 000000000000..33111f903cba --- /dev/null +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/as_map_test.proto @@ -0,0 +1,23 @@ +import "ydb/core/config/protos/marker.proto"; + +package NKikimrConfig; +option java_package = "ru.yandex.kikimr.proto"; + +message SomeTypeForAsMap { + optional string Str = 1; +} + +message MessageWithMap { + option (NMarkers.WithMapType) = true; + repeated SomeTypeForAsMap Entry0 = 1; + repeated SomeTypeForAsMap Entry1 = 2 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry2 = 3 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry3 = 4 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry4 = 5 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry5 = 6 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry6 = 7 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry7 = 8 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry8 = 9 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry9 = 10 [(NMarkers.AsMap) = "SomeMap"]; + repeated SomeTypeForAsMap Entry10 = 111 [(NMarkers.AsMap) = "SomeMap"]; +} diff --git a/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make index 4445e0207a4b..1998395b3360 100644 --- a/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make +++ b/ydb/core/config/tools/protobuf_plugin/ut/protos/ya.make @@ -7,6 +7,7 @@ PEERDIR( SRCS( config_root_test.proto copy_to_test.proto + as_map_test.proto ) CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin) From c882bbd70eedfc7f286d3b9f4509ea1f56f23354 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 9 Feb 2024 17:48:45 +0300 Subject: [PATCH 10/10] Update ydb/public/lib/protobuf/helpers.h Co-authored-by: Ilnaz Nizametdinov --- ydb/public/lib/protobuf/helpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/public/lib/protobuf/helpers.h b/ydb/public/lib/protobuf/helpers.h index 8a187d0ff7a1..30dd802f7747 100644 --- a/ydb/public/lib/protobuf/helpers.h +++ b/ydb/public/lib/protobuf/helpers.h @@ -58,7 +58,7 @@ class TPluginMarkupPrinterGuard private: IPrinter& Printer; - TString PluginName; + const TString PluginName; }; } // namespace NKikimr::NProtobuf