Skip to content

YMQ: a couple of fixes (for stable-24-3) #10082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 5, 2024
Merged

YMQ: a couple of fixes (for stable-24-3) #10082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
70b1086
YMQ: drop empty MD5* fields from JSON responses
qyryq Oct 4, 2024
c790b39
Fix ProtoToJsonPrinter
qyryq Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions ydb/core/http_proxy/json_proto_conversion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@ inline TString ProxyFieldNameConverter(const google::protobuf::FieldDescriptor&

class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
public:
TYdsProtoToJsonPrinter(const google::protobuf::Reflection* reflection,
const NProtobufJson::TProto2JsonConfig& config,
TYdsProtoToJsonPrinter(const NProtobufJson::TProto2JsonConfig& config,
bool skipBase64Encode)
: NProtobufJson::TProto2JsonPrinter(config)
, ProtoReflection(reflection)
, SkipBase64Encode(skipBase64Encode)
{}

Expand Down Expand Up @@ -61,14 +59,15 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
return Base64Encode(str);
};

auto* reflection = proto.GetReflection();
if (field.is_repeated()) {
for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) {
for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) {
PrintStringValue<false>(field, TStringBuf(),
maybeBase64Encode(proto.GetReflection()->GetRepeatedString(proto, &field, i)), json);
maybeBase64Encode(reflection->GetRepeatedString(proto, &field, i)), json);
}
} else {
PrintStringValue<true>(field, key,
maybeBase64Encode(proto.GetReflection()->GetString(proto, &field)), json);
maybeBase64Encode(reflection->GetString(proto, &field)), json);
}
return;
}
Expand All @@ -82,13 +81,14 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
key = MakeKey(field);
}

auto* reflection = proto.GetReflection();
if (field.is_repeated()) {
for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) {
double value = proto.GetReflection()->GetRepeatedInt64(proto, &field, i) / 1000.0;
for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) {
double value = reflection->GetRepeatedInt64(proto, &field, i) / 1000.0;
PrintDoubleValue<false>(TStringBuf(), value, json);
}
} else {
double value = proto.GetReflection()->GetInt64(proto, &field) / 1000.0;
double value = reflection->GetInt64(proto, &field) / 1000.0;
PrintDoubleValue<true>(key, value, json);
}
return;
Expand All @@ -103,19 +103,20 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
key = MakeKey(field);
}

auto* reflection = proto.GetReflection();
if (field.is_repeated()) {
for (int i = 0, endI = ProtoReflection->FieldSize(proto, &field); i < endI; ++i) {
auto value = proto.GetReflection()->GetRepeatedString(proto, &field, i);
for (int i = 0, endI = reflection->FieldSize(proto, &field); i < endI; ++i) {
auto value = reflection->GetRepeatedString(proto, &field, i);
if (!value.empty()) {
PrintStringValue<false>(field, TStringBuf(),
proto.GetReflection()->GetRepeatedString(proto, &field, i), json);
reflection->GetRepeatedString(proto, &field, i), json);
}
}
} else {
auto value = proto.GetReflection()->GetString(proto, &field);
auto value = reflection->GetString(proto, &field);
if (!value.empty()) {
PrintStringValue<true>(field, key,
proto.GetReflection()->GetString(proto, &field), json);
reflection->GetString(proto, &field), json);
}
}
return;
Expand All @@ -126,7 +127,6 @@ class TYdsProtoToJsonPrinter : public NProtobufJson::TProto2JsonPrinter {
}

private:
const google::protobuf::Reflection* ProtoReflection = nullptr;
bool SkipBase64Encode;
};

Expand All @@ -137,7 +137,7 @@ inline void ProtoToJson(const NProtoBuf::Message& resp, NJson::TJsonValue& value
.SetNameGenerator(ProxyFieldNameConverter)
.SetMapAsObject(true)
.SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName);
TYdsProtoToJsonPrinter printer(resp.GetReflection(), config, skipBase64Encode);
TYdsProtoToJsonPrinter printer(config, skipBase64Encode);
printer.Print(resp, *NProtobufJson::CreateJsonMapOutput(value));
}

Expand Down
4 changes: 4 additions & 0 deletions ydb/core/http_proxy/ut/http_proxy_ut.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1970,6 +1970,10 @@ Y_UNIT_TEST_SUITE(TestHttpProxy) {
UNIT_ASSERT(!GetByPath<TString>(succesful0, "MD5OfMessageAttributes").empty());
UNIT_ASSERT(!GetByPath<TString>(succesful0, "MD5OfMessageBody").empty());
UNIT_ASSERT(!GetByPath<TString>(succesful0, "MessageId").empty());

NJson::TJsonValue receiveMessageReq;
receiveMessageReq["QueueUrl"] = resultQueueUrl;
res = SendHttpRequest("/Root", "AmazonSQS.ReceiveMessage", std::move(receiveMessageReq), FormAuthorizationStr("ru-central1"));
}

Y_UNIT_TEST_F(TestDeleteMessageBatch, THttpProxyTestMock) {
Expand Down
12 changes: 12 additions & 0 deletions ydb/core/http_proxy/ut/json_proto_conversion_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,18 @@ Y_UNIT_TEST(ProtoMapToJson) {
}
}

Y_UNIT_TEST(ProtoMapToJson_ReceiveMessageResult) {
// Test using ReceiveMessageResult that has a repeated field with TRANSFORM_BASE64.
// Before fix it failed on messages with attributes.
{
Ydb::Ymq::V1::ReceiveMessageResult message;
message.add_messages()->mutable_message_attributes()->insert({google::protobuf::MapPair<TString, Ydb::Ymq::V1::MessageAttribute>("a", {})});

NJson::TJsonValue jsonObject;
NKikimr::NHttpProxy::ProtoToJson(message, jsonObject, false);
}
}

Y_UNIT_TEST(NlohmannJsonToProtoMap) {
{
nlohmann::json jsonObject;
Expand Down
16 changes: 8 additions & 8 deletions ydb/public/api/protos/draft/ymq.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ message ReceiveMessageResponse {
message Message {
map<string, string> attributes = 1;
string body = 2;
string m_d_5_of_body = 3;
string m_d_5_of_message_attributes = 4;
string m_d_5_of_body = 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string m_d_5_of_message_attributes = 4 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
map<string, MessageAttribute> message_attributes = 5;
string message_id = 6;
string receipt_handle = 7;
Expand All @@ -221,9 +221,9 @@ message SendMessageResponse {
}

message SendMessageResult {
string m_d_5_of_message_attributes = 1;
string m_d_5_of_message_body= 2;
string m_d_5_of_message_system_attributes= 3;
string m_d_5_of_message_attributes = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string m_d_5_of_message_body= 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string m_d_5_of_message_system_attributes= 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string message_id = 4;
string sequence_number = 5;
}
Expand All @@ -248,10 +248,10 @@ message SendMessageBatchRequestEntry {

message SendMessageBatchResultEntry {
string id = 1;
string m_d_5_of_message_body = 2;
string m_d_5_of_message_body = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string message_id = 3;
string m_d_5_of_message_attributes = 4;
string m_d_5_of_message_system_attributes = 5;
string m_d_5_of_message_attributes = 4 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string m_d_5_of_message_system_attributes = 5 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string sequence_number = 6;
}

Expand Down
Loading