From 91e7acf745d64c396e7ca7f4eec454a1964ba21f Mon Sep 17 00:00:00 2001 From: Sergey Skovorodkin Date: Fri, 4 Oct 2024 08:54:04 +0000 Subject: [PATCH 1/2] YMQ: drop empty MD5* fields from JSON responses (cherry picked from commit 9fb3767377607883ae69d0ff674769fad3a969c6) --- ydb/public/api/protos/draft/ymq.proto | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ydb/public/api/protos/draft/ymq.proto b/ydb/public/api/protos/draft/ymq.proto index 22e1a3f9d7e7..b699c597ce81 100644 --- a/ydb/public/api/protos/draft/ymq.proto +++ b/ydb/public/api/protos/draft/ymq.proto @@ -194,8 +194,8 @@ message ReceiveMessageResponse { message Message { map attributes = 1; string body = 2; - string m_d_5_of_body = 3; - string m_d_5_of_message_attributes = 4; + string m_d_5_of_body = 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string m_d_5_of_message_attributes = 4 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; map message_attributes = 5; string message_id = 6; string receipt_handle = 7; @@ -221,9 +221,9 @@ message SendMessageResponse { } message SendMessageResult { - string m_d_5_of_message_attributes = 1; - string m_d_5_of_message_body= 2; - string m_d_5_of_message_system_attributes= 3; + string m_d_5_of_message_attributes = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string m_d_5_of_message_body= 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string m_d_5_of_message_system_attributes= 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string message_id = 4; string sequence_number = 5; } @@ -248,10 +248,10 @@ message SendMessageBatchRequestEntry { message SendMessageBatchResultEntry { string id = 1; - string m_d_5_of_message_body = 2; + string m_d_5_of_message_body = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string message_id = 3; - string m_d_5_of_message_attributes = 4; - string m_d_5_of_message_system_attributes = 5; + string m_d_5_of_message_attributes = 4 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string m_d_5_of_message_system_attributes = 5 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string sequence_number = 6; } From 200f0c7981d693c87ad1afba2b8032a4e7e5f9ef Mon Sep 17 00:00:00 2001 From: Sergey Skovorodkin Date: Fri, 4 Oct 2024 13:04:20 +0000 Subject: [PATCH 2/2] Fix ProtoToJsonPrinter --- ydb/core/http_proxy/json_proto_conversion.h | 32 +++++++++---------- ydb/core/http_proxy/ut/http_proxy_ut.h | 4 +++ .../ut/json_proto_conversion_ut.cpp | 12 +++++++ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/ydb/core/http_proxy/json_proto_conversion.h b/ydb/core/http_proxy/json_proto_conversion.h index 15a9142cc3ee..61e3ff14c20f 100644 --- a/ydb/core/http_proxy/json_proto_conversion.h +++ b/ydb/core/http_proxy/json_proto_conversion.h @@ -23,11 +23,9 @@ inline TString ProxyFieldNameConverter(const google::protobuf::FieldDescriptor& class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter { public: - TYdsProtoToJsonPrinter(const google::protobuf::Reflection* reflection, - const NProtobufJson::TProto2JsonConfig& config, + TYdsProtoToJsonPrinter(const NProtobufJson::TProto2JsonConfig& config, bool skipBase64Encode) : NProtobufJson::TProto2JsonPrinter(config) - , ProtoReflection(reflection) , SkipBase64Encode(skipBase64Encode) {} @@ -61,14 +59,15 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter { return Base64Encode(str); }; + auto* reflection = proto.GetReflection(); if (field.is_repeated()) { - for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) { + for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) { PrintStringValue(field, TStringBuf(), - maybeBase64Encode(proto.GetReflection()->GetRepeatedString(proto, &field, i)), json); + maybeBase64Encode(reflection->GetRepeatedString(proto, &field, i)), json); } } else { PrintStringValue(field, key, - maybeBase64Encode(proto.GetReflection()->GetString(proto, &field)), json); + maybeBase64Encode(reflection->GetString(proto, &field)), json); } return; } @@ -82,13 +81,14 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter { key = MakeKey(field); } + auto* reflection = proto.GetReflection(); if (field.is_repeated()) { - for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) { - double value = proto.GetReflection()->GetRepeatedInt64(proto, &field, i) / 1000.0; + for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) { + double value = reflection->GetRepeatedInt64(proto, &field, i) / 1000.0; PrintDoubleValue(TStringBuf(), value, json); } } else { - double value = proto.GetReflection()->GetInt64(proto, &field) / 1000.0; + double value = reflection->GetInt64(proto, &field) / 1000.0; PrintDoubleValue(key, value, json); } return; @@ -103,19 +103,20 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter { key = MakeKey(field); } + auto* reflection = proto.GetReflection(); if (field.is_repeated()) { - for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) { - auto value = proto.GetReflection()->GetRepeatedString(proto, &field, i); + for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) { + auto value = reflection->GetRepeatedString(proto, &field, i); if (!value.empty()) { PrintStringValue(field, TStringBuf(), - proto.GetReflection()->GetRepeatedString(proto, &field, i), json); + reflection->GetRepeatedString(proto, &field, i), json); } } } else { - auto value = proto.GetReflection()->GetString(proto, &field); + auto value = reflection->GetString(proto, &field); if (!value.empty()) { PrintStringValue(field, key, - proto.GetReflection()->GetString(proto, &field), json); + reflection->GetString(proto, &field), json); } } return; @@ -126,7 +127,6 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter { } private: - const google::protobuf::Reflection* ProtoReflection = nullptr; bool SkipBase64Encode; }; @@ -137,7 +137,7 @@ inline void ProtoToJson(const NProtoBuf::Message& resp, NJson::TJsonValue& value .SetNameGenerator(ProxyFieldNameConverter) .SetMapAsObject(true) .SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName); - TYdsProtoToJsonPrinter printer(resp.GetReflection(), config, skipBase64Encode); + TYdsProtoToJsonPrinter printer(config, skipBase64Encode); printer.Print(resp, *NProtobufJson::CreateJsonMapOutput(value)); } diff --git a/ydb/core/http_proxy/ut/http_proxy_ut.h b/ydb/core/http_proxy/ut/http_proxy_ut.h index 006037e26894..ec29797a62a6 100644 --- a/ydb/core/http_proxy/ut/http_proxy_ut.h +++ b/ydb/core/http_proxy/ut/http_proxy_ut.h @@ -1970,6 +1970,10 @@ Y_UNIT_TEST_SUITE(TestHttpProxy) { UNIT_ASSERT(!GetByPath(succesful0, "MD5OfMessageAttributes").empty()); UNIT_ASSERT(!GetByPath(succesful0, "MD5OfMessageBody").empty()); UNIT_ASSERT(!GetByPath(succesful0, "MessageId").empty()); + + NJson::TJsonValue receiveMessageReq; + receiveMessageReq["QueueUrl"] = resultQueueUrl; + res = SendHttpRequest("/Root", "AmazonSQS.ReceiveMessage", std::move(receiveMessageReq), FormAuthorizationStr("ru-central1")); } Y_UNIT_TEST_F(TestDeleteMessageBatch, THttpProxyTestMock) { diff --git a/ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp b/ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp index 3f898280de0b..dd117d0535cf 100644 --- a/ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp +++ b/ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp @@ -202,6 +202,18 @@ Y_UNIT_TEST(ProtoMapToJson) { } } +Y_UNIT_TEST(ProtoMapToJson_ReceiveMessageResult) { + // Test using ReceiveMessageResult that has a repeated field with TRANSFORM_BASE64. + // Before fix it failed on messages with attributes. + { + Ydb::Ymq::V1::ReceiveMessageResult message; + message.add_messages()->mutable_message_attributes()->insert({google::protobuf::MapPair("a", {})}); + + NJson::TJsonValue jsonObject; + NKikimr::NHttpProxy::ProtoToJson(message, jsonObject, false); + } +} + Y_UNIT_TEST(NlohmannJsonToProtoMap) { { nlohmann::json jsonObject;