Skip to content

Fix binary docvalue_fields with padding #70826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 26, 2021
Merged

Fix binary docvalue_fields with padding #70826

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0f5de93
Fix binary docvalue_fields with padding
mayya-sharipova Mar 24, 2021
a6ea516
Merge remote-tracking branch 'upstream/master' into binary-fields_doc…
mayya-sharipova Mar 26, 2021
73be1d9
Address Nik's feedback
mayya-sharipova Mar 26, 2021
3aa1286
Address Nik's feedback 2
mayya-sharipova Mar 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions ...ss/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,13 @@

- do:
search:
rest_total_hits_as_int: true
body:
script_fields:
field:
field1:
script:
source: "doc['binary'].get(0).utf8ToString()"
- match: { hits.hits.0.fields.field.0: "Some binary blob" }

- do:
search:
rest_total_hits_as_int: true
body:
script_fields:
field:
field2:
script:
source: "doc['binary'].value.utf8ToString()"
- match: { hits.hits.0.fields.field.0: "Some binary blob" }
- match: { hits.hits.0.fields.field1.0: "Some binary blob" }
- match: { hits.hits.0.fields.field2.0: "Some binary blob" }
49 changes: 49 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/search/350_binary_field.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
---
"binary":
- skip:
features: ["headers"]
version: " - 7.99.99"
reason: "docvalues_fields on binary field were corrected in 8.0"
- do:
indices.create:
index: test
body:
mappings:
properties:
binary:
type: binary
doc_values: true

- do:
#other formats (e.g. cbor) may not support parsing of binary
headers:
Content-Type: application/json
index:
index: test
id: 1
body:
binary: U29tZSBiaW5hcnkgYmxvYg==

- do:
indices.refresh: {}

- do:
search:
index: test
body:
docvalue_fields: [ "binary" ]
- match: { hits.hits.0.fields.binary.0: "U29tZSBiaW5hcnkgYmxvYg==" }

- do:
search:
index: test
body:
fields: [ "binary" ]
- match: { hits.hits.0.fields.binary.0: "U29tZSBiaW5hcnkgYmxvYg==" }

- do:
search:
index: test
body:
_source: ["binary"]
- match: { hits.hits.0._source.binary: "U29tZSBiaW5hcnkgYmxvYg==" }
4 changes: 2 additions & 2 deletions server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -810,7 +810,7 @@ public void testDocValueFields() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ="));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));

builder = client().prepareSearch().setQuery(matchAllQuery())
Expand All @@ -835,7 +835,7 @@ public void testDocValueFields() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ="));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));

builder = client().prepareSearch().setQuery(matchAllQuery())
Expand Down
1 change: 0 additions & 1 deletion server/src/main/java/org/elasticsearch/search/DocValueFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@ public void writeTo(StreamOutput out) {
@Override
public String format(BytesRef value) {
return Base64.getEncoder()
.withoutPadding()
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
}

Expand Down
4 changes: 2 additions & 2 deletions server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,10 @@ public void testRawFormat() {

public void testBinaryFormat() {
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));
assertEquals("KmQ=", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));

assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ="));
}

public void testBooleanFormat() {
Expand Down