Skip to content

Commit 4eceb69

Browse files
author
Vadim Averin
authored
YDB-2321 Add subnet operations in IP UDF (#981)
* Add basic subnet functions in IP UDF * Minor fixes * Small fixes * Post-merge fixes * Add subnet match * Add GetSubnetByMask * Add & canonize tests * Add RU docs for subnet ops * Add EN docs for subnet ops * Revert library changes * Add usage clarification * Add usage examples for IP subnet ops * Add usage examples for IP subnet ops (RU) * Fix codestyle * Remove unnecessary string allocation
1 parent 2547396 commit 4eceb69

File tree

8 files changed

+465
-30
lines changed

8 files changed

+465
-30
lines changed

‎ydb/docs/en/core/yql/reference/yql-core/udf/list/ip.md

+19
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,17 @@ The `Ip` module supports both the IPv4 and IPv6 addresses. By default, they are
55
**List of functions**
66

77
* ```Ip::FromString(String{Flags:AutoMap}) -> String?``` - From a human-readable representation to a binary representation.
8+
* ```Ip::SubnetFromString(String{Flags:AutoMap}) -> String?``` - From a human-readable representation of subnet to a binary representation.
89
* ```Ip::ToString(String{Flags:AutoMap}) -> String?``` - From a binary representation to a human-readable representation.
10+
* ```Ip::SubnetToString(String{Flags:AutoMap}) -> String?``` - From a binary representation of subnet to a human-readable representation.
911
* ```Ip::IsIPv4(String?) -> Bool```
1012
* ```Ip::IsIPv6(String?) -> Bool```
1113
* ```Ip::IsEmbeddedIPv4(String?) -> Bool```
1214
* ```Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String```: IPv6 remains unchanged, and IPv4 becomes embedded in IPv6
1315
* ```Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String```: The second argument is the subnet size, by default it's 24 for IPv4 and 64 for IPv6
16+
* ```Ip::GetSubnetByMask(String{Flags:AutoMap}, String{Flags:AutoMap}) -> String```: The first argument is the base address, the second argument is the bit mask of a desired subnet.
17+
* ```Ip::SubnetMatch(String{Flags:AutoMap}, String{Flags:AutoMap}) -> Bool```: The first argument is a subnet, the second argument is a subnet or an address.
18+
1419

1520
**Examples**
1621

@@ -25,5 +30,19 @@ SELECT
2530
Ip::FromString("213.180.193.3")
2631
)
2732
); -- "213.180.193.0"
33+
34+
SELECT
35+
Ip::SubnetMatch(
36+
Ip::SubnetFromString("192.168.0.1/16"),
37+
Ip::FromString("192.168.1.14"),
38+
); -- true
39+
40+
SELECT
41+
Ip::ToString(
42+
Ip::GetSubnetByMask(
43+
Ip::FromString("192.168.0.1"),
44+
Ip::FromString("255.255.0.0")
45+
)
46+
); -- "192.168.0.0"
2847
```
2948

‎ydb/docs/ru/core/yql/reference/yql-core/udf/list/ip.md

+18
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,16 @@
44
**Список функций**
55

66
* ```Ip::FromString(String{Flags:AutoMap}) -> String?``` - из человекочитаемого представления в бинарное
7+
* ```Ip::SubnetFromString(String{Flags:AutoMap}) -> String?``` - из человекочитаемого представления подсети в бинарное
78
* ```Ip::ToString(String{Flags:AutoMap}) -> String?``` - из бинарного представления в человекочитаемое
9+
* ```Ip::ToString(String{Flags:AutoMap}) -> String?``` - из бинарного представления подсети в человекочитаемое
810
* ```Ip::IsIPv4(String?) -> Bool```
911
* ```Ip::IsIPv6(String?) -> Bool```
1012
* ```Ip::IsEmbeddedIPv4(String?) -> Bool```
1113
* ```Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String``` - IPv6 остается без изменений, а IPv4 становится embedded в IPv6
1214
* ```Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String``` - во втором аргументе размер подсети, по умолчанию 24 для IPv4 и 64 для IPv6
15+
* ```Ip::GetSubnetByMask(String{Flags:AutoMap}, String{Flags:AutoMap}) -> String``` - во втором аргументе битовая маска подсети
16+
* ```Ip::SubnetMatch(String{Flags:AutoMap}, String{Flags:AutoMap}) -> Bool``` - в первом аргументе подсеть, во втором аргументе подсеть или адрес
1317

1418
**Примеры**
1519

@@ -24,4 +28,18 @@ SELECT
2428
Ip::FromString("213.180.193.3")
2529
)
2630
); -- "213.180.193.0"
31+
32+
SELECT
33+
Ip::SubnetMatch(
34+
Ip::SubnetFromString("192.168.0.1/16"),
35+
Ip::FromString("192.168.1.14"),
36+
); -- true
37+
38+
SELECT
39+
Ip::ToString(
40+
Ip::GetSubnetByMask(
41+
Ip::FromString("192.168.0.1"),
42+
Ip::FromString("255.255.0.0")
43+
)
44+
); -- "192.168.0.0"
2745
```

‎ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h

+217-30
Original file line numberDiff line numberDiff line change
@@ -14,67 +14,214 @@ namespace {
1414
using TUnboxedValue = NKikimr::NUdf::TUnboxedValue;
1515
using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod;
1616

17+
ui8 GetAddressRangePrefix(const TIpAddressRange& range) {
18+
if (range.Contains(TIpv6Address(ui128(0), TIpv6Address::Ipv6)) && range.Contains(TIpv6Address(ui128(-1), TIpv6Address::Ipv6))) {
19+
return 0;
20+
}
21+
if (range.Size() == 0) {
22+
return range.Type() == TIpv6Address::Ipv4 ? 32 : 128;
23+
}
24+
ui128 size = range.Size();
25+
size_t sizeLog = MostSignificantBit(size);
26+
return ui8((range.Type() == TIpv6Address::Ipv4 ? 32 : 128) - sizeLog);
27+
}
28+
1729
struct TRawIp4 {
1830
ui8 a, b, c, d;
31+
32+
static TRawIp4 FromIpAddress(const TIpv6Address& addr) {
33+
ui128 x = addr;
34+
return {
35+
ui8(x >> 24 & 0xff),
36+
ui8(x >> 16 & 0xff),
37+
ui8(x >> 8 & 0xff),
38+
ui8(x & 0xff)
39+
};
40+
}
41+
42+
static TRawIp4 MaskFromPrefix(ui8 prefix) {
43+
ui128 x = ui128(-1) << int(32 - prefix);
44+
x &= ui128(ui32(-1));
45+
return FromIpAddress({x, TIpv6Address::Ipv4});
46+
}
47+
48+
TIpv6Address ToIpAddress() const {
49+
return {a, b, c, d};
50+
}
51+
52+
std::pair<TRawIp4, TRawIp4> ApplyMask(const TRawIp4& mask) const {
53+
return {{
54+
ui8(a & mask.a),
55+
ui8(b & mask.b),
56+
ui8(c & mask.c),
57+
ui8(d & mask.d)
58+
},{
59+
ui8(a | ~mask.a),
60+
ui8(b | ~mask.b),
61+
ui8(c | ~mask.c),
62+
ui8(d | ~mask.d)
63+
}};
64+
}
65+
};
66+
67+
struct TRawIp4Subnet {
68+
TRawIp4 base, mask;
69+
70+
static TRawIp4Subnet FromIpRange(const TIpAddressRange& range) {
71+
return {TRawIp4::FromIpAddress(*range.Begin()), TRawIp4::MaskFromPrefix(GetAddressRangePrefix(range))};
72+
}
73+
74+
TIpAddressRange ToIpRange() const {
75+
auto range = base.ApplyMask(mask);
76+
return {range.first.ToIpAddress(), range.second.ToIpAddress()};
77+
}
1978
};
2079

2180
struct TRawIp6 {
2281
ui8 a1, a0, b1, b0, c1, c0, d1, d0, e1, e0, f1, f0, g1, g0, h1, h0;
82+
83+
static TRawIp6 FromIpAddress(const TIpv6Address& addr) {
84+
ui128 x = addr;
85+
return {
86+
ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff),
87+
ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff),
88+
ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff),
89+
ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff),
90+
ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff),
91+
ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff),
92+
ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff),
93+
ui8(x >> 8 & 0xff), ui8(x & 0xff)
94+
};
95+
}
96+
97+
static TRawIp6 MaskFromPrefix(ui8 prefix) {
98+
ui128 x = ui128(-1) << int(128 - prefix);
99+
if (prefix == 0) x = 0;
100+
return FromIpAddress({x, TIpv6Address::Ipv6});
101+
}
102+
103+
TIpv6Address ToIpAddress() const {
104+
return {ui16(ui32(a1) << ui32(8) | ui32(a0)),
105+
ui16(ui32(b1) << ui32(8) | ui32(b0)),
106+
ui16(ui32(c1) << ui32(8) | ui32(c0)),
107+
ui16(ui32(d1) << ui32(8) | ui32(d0)),
108+
ui16(ui32(e1) << ui32(8) | ui32(e0)),
109+
ui16(ui32(f1) << ui32(8) | ui32(f0)),
110+
ui16(ui32(g1) << ui32(8) | ui32(g0)),
111+
ui16(ui32(h1) << ui32(8) | ui32(h0)),
112+
};
113+
}
114+
115+
std::pair<TRawIp6, TRawIp6> ApplyMask(const TRawIp6& mask) const {
116+
return { {
117+
ui8(a1 & mask.a1),
118+
ui8(a0 & mask.a0),
119+
ui8(b1 & mask.b1),
120+
ui8(b0 & mask.b0),
121+
ui8(c1 & mask.c1),
122+
ui8(c0 & mask.c0),
123+
ui8(d1 & mask.d1),
124+
ui8(d0 & mask.d0),
125+
ui8(e1 & mask.e1),
126+
ui8(e0 & mask.e0),
127+
ui8(f1 & mask.f1),
128+
ui8(f0 & mask.f0),
129+
ui8(g1 & mask.g1),
130+
ui8(g0 & mask.g0),
131+
ui8(h1 & mask.h1),
132+
ui8(h0 & mask.h0)
133+
}, {
134+
ui8(a1 | ~mask.a1),
135+
ui8(a0 | ~mask.a0),
136+
ui8(b1 | ~mask.b1),
137+
ui8(b0 | ~mask.b0),
138+
ui8(c1 | ~mask.c1),
139+
ui8(c0 | ~mask.c0),
140+
ui8(d1 | ~mask.d1),
141+
ui8(d0 | ~mask.d0),
142+
ui8(e1 | ~mask.e1),
143+
ui8(e0 | ~mask.e0),
144+
ui8(f1 | ~mask.f1),
145+
ui8(f0 | ~mask.f0),
146+
ui8(g1 | ~mask.g1),
147+
ui8(g0 | ~mask.g0),
148+
ui8(h1 | ~mask.h1),
149+
ui8(h0 | ~mask.h0)
150+
}};
151+
}
152+
};
153+
154+
struct TRawIp6Subnet {
155+
TRawIp6 base, mask;
156+
157+
static TRawIp6Subnet FromIpRange(const TIpAddressRange& range) {
158+
return {TRawIp6::FromIpAddress(*range.Begin()), TRawIp6::MaskFromPrefix(GetAddressRangePrefix(range))};
159+
}
160+
161+
TIpAddressRange ToIpRange() const {
162+
auto range = base.ApplyMask(mask);
163+
return {range.first.ToIpAddress(), range.second.ToIpAddress()};
164+
}
23165
};
24166

25167
TIpv6Address DeserializeAddress(const TStringRef& str) {
26168
TIpv6Address addr;
27169
if (str.Size() == 4) {
28170
TRawIp4 addr4;
29171
memcpy(&addr4, str.Data(), sizeof addr4);
30-
addr = {addr4.a, addr4.b, addr4.c, addr4.d};
172+
addr = addr4.ToIpAddress();
31173
} else if (str.Size() == 16) {
32174
TRawIp6 addr6;
33175
memcpy(&addr6, str.Data(), sizeof addr6);
34-
addr = {ui16(ui32(addr6.a1) << ui32(8) | ui32(addr6.a0)),
35-
ui16(ui32(addr6.b1) << ui32(8) | ui32(addr6.b0)),
36-
ui16(ui32(addr6.c1) << ui32(8) | ui32(addr6.c0)),
37-
ui16(ui32(addr6.d1) << ui32(8) | ui32(addr6.d0)),
38-
ui16(ui32(addr6.e1) << ui32(8) | ui32(addr6.e0)),
39-
ui16(ui32(addr6.f1) << ui32(8) | ui32(addr6.f0)),
40-
ui16(ui32(addr6.g1) << ui32(8) | ui32(addr6.g0)),
41-
ui16(ui32(addr6.h1) << ui32(8) | ui32(addr6.h0)),
42-
};
176+
addr = addr6.ToIpAddress();
43177
} else {
44178
ythrow yexception() << "Incorrect size of input, expected "
45179
<< "4 or 16, got " << str.Size();
46180
}
47181
return addr;
48182
}
49183

184+
TIpAddressRange DeserializeSubnet(const TStringRef& str) {
185+
TIpAddressRange range;
186+
if (str.Size() == sizeof(TRawIp4Subnet)) {
187+
TRawIp4Subnet subnet4;
188+
memcpy(&subnet4, str.Data(), sizeof subnet4);
189+
range = subnet4.ToIpRange();
190+
} else if (str.Size() == sizeof(TRawIp6Subnet)) {
191+
TRawIp6Subnet subnet6;
192+
memcpy(&subnet6, str.Data(), sizeof subnet6);
193+
range = subnet6.ToIpRange();
194+
} else {
195+
ythrow yexception() << "Invalid binary representation";
196+
}
197+
return range;
198+
}
199+
50200
TString SerializeAddress(const TIpv6Address& addr) {
51201
Y_ENSURE(addr.Type() == TIpv6Address::Ipv4 || addr.Type() == TIpv6Address::Ipv6);
52202
TString res;
53-
ui128 x = addr;
54203
if (addr.Type() == TIpv6Address::Ipv4) {
55-
TRawIp4 addr4 {
56-
ui8(x >> 24 & 0xff),
57-
ui8(x >> 16 & 0xff),
58-
ui8(x >> 8 & 0xff),
59-
ui8(x & 0xff)
60-
};
204+
auto addr4 = TRawIp4::FromIpAddress(addr);
61205
res = TString(reinterpret_cast<const char *>(&addr4), sizeof addr4);
62206
} else if (addr.Type() == TIpv6Address::Ipv6) {
63-
TRawIp6 addr6 {
64-
ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff),
65-
ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff),
66-
ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff),
67-
ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff),
68-
ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff),
69-
ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff),
70-
ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff),
71-
ui8(x >> 8 & 0xff), ui8(x & 0xff)
72-
};
207+
auto addr6 = TRawIp6::FromIpAddress(addr);
73208
res = TString(reinterpret_cast<const char *>(&addr6), sizeof addr6);
74209
}
75210
return res;
76211
}
77212

213+
TString SerializeSubnet(const TIpAddressRange& range) {
214+
TString res;
215+
if (range.Type() == TIpv6Address::Ipv4) {
216+
auto subnet4 = TRawIp4Subnet::FromIpRange(range);
217+
res = TString(reinterpret_cast<const char *>(&subnet4), sizeof subnet4);
218+
} else if (range.Type() == TIpv6Address::Ipv6) {
219+
auto subnet6 = TRawIp6Subnet::FromIpRange(range);
220+
res = TString(reinterpret_cast<const char *>(&subnet6), sizeof subnet6);
221+
}
222+
return res;
223+
}
224+
78225
SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) {
79226
TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef());
80227
if (addr.Type() != TIpv6Address::Ipv4 && addr.Type() != TIpv6Address::Ipv6) {
@@ -83,10 +230,37 @@ namespace {
83230
return valueBuilder->NewString(SerializeAddress(addr));
84231
}
85232

233+
SIMPLE_STRICT_UDF(TSubnetFromString, TOptionalString(TAutoMapString)) {
234+
TIpAddressRange range = TIpAddressRange::FromCompactString(args[0].AsStringRef());
235+
auto res = SerializeSubnet(range);
236+
return res ? valueBuilder->NewString(res) : TUnboxedValue(TUnboxedValuePod());
237+
}
238+
86239
SIMPLE_UDF(TToString, char*(TAutoMapString)) {
87240
return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false));
88241
}
89242

243+
SIMPLE_UDF(TSubnetToString, char*(TAutoMapString)) {
244+
TStringBuilder result;
245+
auto range = DeserializeSubnet(args[0].AsStringRef());
246+
result << (*range.Begin()).ToString(false);
247+
result << '/';
248+
result << ToString(GetAddressRangePrefix(range));
249+
return valueBuilder->NewString(result);
250+
}
251+
252+
SIMPLE_UDF(TSubnetMatch, bool(TAutoMapString, TAutoMapString)) {
253+
Y_UNUSED(valueBuilder);
254+
auto range1 = DeserializeSubnet(args[0].AsStringRef());
255+
if (args[1].AsStringRef().Size() == sizeof(TRawIp4) || args[1].AsStringRef().Size() == sizeof(TRawIp6)) {
256+
auto addr2 = DeserializeAddress(args[1].AsStringRef());
257+
return TUnboxedValuePod(range1.Contains(addr2));
258+
} else { // second argument is a whole subnet, not a single address
259+
auto range2 = DeserializeSubnet(args[1].AsStringRef());
260+
return TUnboxedValuePod(range1.Contains(range2));
261+
}
262+
}
263+
90264
SIMPLE_STRICT_UDF(TIsIPv4, bool(TOptionalString)) {
91265
Y_UNUSED(valueBuilder);
92266
bool result = false;
@@ -159,14 +333,27 @@ namespace {
159333
return valueBuilder->NewString(SerializeAddress(beg));
160334
}
161335

336+
SIMPLE_UDF(TGetSubnetByMask, char*(TAutoMapString, TAutoMapString)) {
337+
const auto refBase = args[0].AsStringRef();
338+
const auto refMask = args[1].AsStringRef();
339+
TIpv6Address addrBase = DeserializeAddress(refBase);
340+
TIpv6Address addrMask = DeserializeAddress(refMask);
341+
if (addrBase.Type() != addrMask.Type()) {
342+
ythrow yexception() << "Base and mask differ in length";
343+
}
344+
return valueBuilder->NewString(SerializeAddress(TIpv6Address(ui128(addrBase) & ui128(addrMask), addrBase.Type())));
345+
}
346+
162347
#define EXPORTED_IP_BASE_UDF \
163348
TFromString, \
349+
TSubnetFromString, \
164350
TToString, \
351+
TSubnetToString, \
165352
TIsIPv4, \
166353
TIsIPv6, \
167354
TIsEmbeddedIPv4, \
168355
TConvertToIPv6, \
169-
TGetSubnet
356+
TGetSubnet, \
357+
TSubnetMatch, \
358+
TGetSubnetByMask
170359
}
171-
172-

‎ydb/library/yql/udfs/common/ip_base/test/canondata/result.json

+5
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,10 @@
33
{
44
"uri": "file://test.test_Basic_/results.txt"
55
}
6+
],
7+
"test.test[Subnets]": [
8+
{
9+
"uri": "file://test.test_Subnets_/results.txt"
10+
}
611
]
712
}

0 commit comments

Comments
 (0)