Skip to content

YDB-2321 Refactor IP UDF to use supported ipmath library #618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jan 18, 2024
Merged

YDB-2321 Refactor IP UDF to use supported ipmath library #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
66c888d
Apply patch with changes
Dec 21, 2023
4ee2858
Add & canonize some tests
Dec 27, 2023
15c6cfd
More refactoring
Jan 17, 2024
f39371e
Final cleanup
Jan 17, 2024
0c3ba9e
Apply patch with changes
Dec 21, 2023
0136b9c
Add & canonize some tests
Dec 27, 2023
27daa3c
More refactoring
Jan 17, 2024
ba6dbad
Final cleanup
Jan 17, 2024
24ca2e6
Add explicit dependencies on library/cpp/ipv6_address
Jan 17, 2024
7b72ddc
Merge branch 'ydb-platform:main' into ip-udf-refactoring
avevad Jan 17, 2024
7f2f82b
Merge remote-tracking branch 'origin/ip-udf-refactoring' into ip-udf-…
Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 78 additions & 53 deletions ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

#include <ydb/library/yql/public/udf/udf_helpers.h>

#include <util/draft/ip.h>
#include <library/cpp/ipv6_address/ipv6_address.h>
#include <library/cpp/ipmath/ipmath.h>
#include <util/generic/buffer.h>

namespace {
Expand All @@ -13,39 +14,77 @@ namespace {
using TUnboxedValue = NKikimr::NUdf::TUnboxedValue;
using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod;

struct TSerializeIpVisitor {
TStringRef operator()(const TIp4& ip) const {
return TStringRef(reinterpret_cast<const char*>(&ip), 4);
struct TRawIp4 {
ui8 a, b, c, d;
};

struct TRawIp6 {
ui8 a1, a0, b1, b0, c1, c0, d1, d0, e1, e0, f1, f0, g1, g0, h1, h0;
};

TIpv6Address DeserializeAddress(const TStringRef& str) {
TIpv6Address addr;
if (str.Size() == 4) {
TRawIp4 addr4;
memcpy(&addr4, str.Data(), sizeof addr4);
addr = {addr4.a, addr4.b, addr4.c, addr4.d};
} else if (str.Size() == 16) {
TRawIp6 addr6;
memcpy(&addr6, str.Data(), sizeof addr6);
addr = {ui16(ui32(addr6.a1) << ui32(8) | ui32(addr6.a0)),
ui16(ui32(addr6.b1) << ui32(8) | ui32(addr6.b0)),
ui16(ui32(addr6.c1) << ui32(8) | ui32(addr6.c0)),
ui16(ui32(addr6.d1) << ui32(8) | ui32(addr6.d0)),
ui16(ui32(addr6.e1) << ui32(8) | ui32(addr6.e0)),
ui16(ui32(addr6.f1) << ui32(8) | ui32(addr6.f0)),
ui16(ui32(addr6.g1) << ui32(8) | ui32(addr6.g0)),
ui16(ui32(addr6.h1) << ui32(8) | ui32(addr6.h0)),
};
} else {
ythrow yexception() << "Incorrect size of input, expected "
<< "4 or 16, got " << str.Size();
}
TStringRef operator()(const TIp6& ip) const {
return TStringRef(reinterpret_cast<const char*>(&ip.Data), 16);
return addr;
}

TString SerializeAddress(const TIpv6Address& addr) {
Y_ENSURE(addr.Type() == TIpv6Address::Ipv4 || addr.Type() == TIpv6Address::Ipv6);
TString res;
ui128 x = addr;
if (addr.Type() == TIpv6Address::Ipv4) {
TRawIp4 addr4 {
ui8(x >> 24 & 0xff),
ui8(x >> 16 & 0xff),
ui8(x >> 8 & 0xff),
ui8(x & 0xff)
};
res = TString(reinterpret_cast<const char *>(&addr4), sizeof addr4);
} else if (addr.Type() == TIpv6Address::Ipv6) {
TRawIp6 addr6 {
ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff),
ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff),
ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff),
ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff),
ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff),
ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff),
ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff),
ui8(x >> 8 & 0xff), ui8(x & 0xff)
};
res = TString(reinterpret_cast<const char *>(&addr6), sizeof addr6);
}
};
return res;
}

SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) {
try {
TString input(args[0].AsStringRef());
const TIp4Or6& ip = Ip4Or6FromString(input.c_str());
return valueBuilder->NewString(std::visit(TSerializeIpVisitor(), ip));
} catch (TSystemError&) {
TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef());
if (addr.Type() != TIpv6Address::Ipv4 && addr.Type() != TIpv6Address::Ipv6) {
return TUnboxedValue();
}
return valueBuilder->NewString(SerializeAddress(addr));
}

SIMPLE_UDF(TToString, char*(TAutoMapString)) {
const auto& ref = args[0].AsStringRef();
if (ref.Size() == 4) {
TIp4 ip;
memcpy(&ip, ref.Data(), sizeof(ip));
return valueBuilder->NewString(Ip4Or6ToString(ip));
} else if (ref.Size() == 16) {
TIp6 ip;
memcpy(&ip.Data, ref.Data(), sizeof(ip.Data));
return valueBuilder->NewString(Ip4Or6ToString(ip));
} else {
ythrow yexception() << "Incorrect size of input, expected "
<< "4 or 16, got " << ref.Size();
}
return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false));
}

SIMPLE_STRICT_UDF(TIsIPv4, bool(TOptionalString)) {
Expand Down Expand Up @@ -73,15 +112,8 @@ namespace {
bool result = false;
if (args[0]) {
const auto ref = args[0].AsStringRef();
if (ref.Size() == 16 && ref.Data()[10] == -1) {
bool allZeroes = true;
for (int i = 0; i < 10; ++i) {
if (ref.Data()[i] != 0) {
allZeroes = false;
break;
}
}
result = allZeroes;
if (ref.Size() == 16) {
result = DeserializeAddress(ref).Isv4MappedTov6();
}
}
return TUnboxedValuePod(result);
Expand All @@ -92,10 +124,9 @@ namespace {
if (ref.Size() == 16) {
return valueBuilder->NewString(ref);
} else if (ref.Size() == 4) {
TIp4 ipv4;
memcpy(&ipv4, ref.Data(), sizeof(ipv4));
const TIp6 ipv6 = Ip6FromIp4(ipv4);
return valueBuilder->NewString(TStringRef(reinterpret_cast<const char*>(&ipv6.Data), 16));
TIpv6Address addr4 = DeserializeAddress(ref);
auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6);
return valueBuilder->NewString(SerializeAddress(addr6));
} else {
ythrow yexception() << "Incorrect size of input, expected "
<< "4 or 16, got " << ref.Size();
Expand All @@ -105,33 +136,27 @@ namespace {
SIMPLE_UDF_WITH_OPTIONAL_ARGS(TGetSubnet, char*(TAutoMapString, TOptionalByte), 1) {
const auto ref = args[0].AsStringRef();
ui8 subnetSize = args[1].GetOrDefault<ui8>(0);

TIpv6Address addr = DeserializeAddress(ref);
if (ref.Size() == 4) {
if (!subnetSize) {
subnetSize = 24;
}
if (subnetSize > 32) {
subnetSize = 32;
}
} else if (ref.Size() == 16) {
if (!subnetSize) {
subnetSize = 64;
}
if (subnetSize > 128) {
subnetSize = 128;
}
} else {
ythrow yexception() << "Incorrect size of input, expected "
<< "4 or 16, got " << ref.Size();
}
TBuffer result(ref.Data(), ref.Size());
int bytesToMask = ref.Size() * 8 - subnetSize;
ui8 currentByte = ref.Size() - 1;
while (bytesToMask > 0) {
if (bytesToMask > 8) {
result.Data()[currentByte] = 0;
} else {
result.Data()[currentByte] = result.Data()[currentByte] & (0xff << bytesToMask);
}
bytesToMask -= 8;
--currentByte;
}

return valueBuilder->NewString(TStringRef(result.Data(), result.Size()));
TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize);
return valueBuilder->NewString(SerializeAddress(beg));
}

#define EXPORTED_IP_BASE_UDF \
Expand Down
2 changes: 2 additions & 0 deletions ydb/library/yql/udfs/common/ip_base/lib/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ SRCS(

PEERDIR(
ydb/library/yql/public/udf
library/cpp/ipmath
library/cpp/ipv6_address
)

END()
93 changes: 93 additions & 0 deletions ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,26 @@
"String"
]
]
];
[
"single_subnet4";
[
"OptionalType";
[
"DataType";
"String"
]
]
];
[
"single_subnet6";
[
"OptionalType";
[
"DataType";
"String"
]
]
]
]
]
Expand All @@ -113,6 +133,12 @@
];
[
"127.0.0.0"
];
[
"127.0.0.1"
];
[
"127.0.0.1"
]
];
[
Expand All @@ -136,6 +162,12 @@
];
[
"::"
];
[
"::"
];
[
"::1"
]
];
[
Expand All @@ -161,6 +193,12 @@
];
[
"213.180.0.0"
];
[
"213.180.193.3"
];
[
"213.180.193.3"
]
];
[
Expand All @@ -186,6 +224,12 @@
];
[
"2a02::"
];
[
"2a02:6b8::"
];
[
"2a02:6b8::3"
]
];
[
Expand All @@ -211,6 +255,12 @@
];
[
"2400::"
];
[
"2400:cb00::"
];
[
"2400:cb00:2048:1::681c:1b65"
]
];
[
Expand All @@ -236,6 +286,12 @@
];
[
"fe80::"
];
[
"fe80::"
];
[
"fe80::215:b2ff:fea9:67ce"
]
];
[
Expand All @@ -261,6 +317,12 @@
];
[
"::"
];
[
"::"
];
[
"::ffff:77.75.155.3"
]
];
[
Expand All @@ -272,7 +334,38 @@
#;
#;
#;
#;
#;
#
];
[
[
"\0\0\0\0"
];
[
"0.0.0.0"
];
%true;
%false;
%false;
[
"::ffff:0.0.0.0"
];
[
"0.0.0.0"
];
[
"0.0.0.0"
];
[
"0.0.0.0"
];
[
"0.0.0.0"
];
[
"0.0.0.0"
]
]
]
}
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@
{"key"="fe80::215:b2ff:fea9:67ce";"subkey"="";"value"=""};
{"key"="::ffff:77.75.155.3";"subkey"="";"value"=""};
{"key"="sdfsdfsdf";"subkey"="";"value"=""};
{"key"="0.0.0.0";"subkey"="";value=""};
4 changes: 3 additions & 1 deletion ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ SELECT
Ip::ToString(Ip::ConvertToIPv6(internal_representation)) AS all_ipv6,
Ip::ToString(Ip::GetSubnet(internal_representation)) AS default_subnet,
Ip::ToString(Ip::GetSubnet(internal_representation, 125)) AS small_subnet,
Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet
Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet,
Ip::ToString(Ip::GetSubnet(internal_representation, 32)) AS single_subnet4,
Ip::ToString(Ip::GetSubnet(internal_representation, 128)) AS single_subnet6
FROM (
SELECT Ip::FromString(key) AS internal_representation FROM Input
);