Skip to content

Commit e5a10e9

Browse files
jasontedorjasontedor
committed
Merge pull request elastic#13907 from jasontedor/hash-be-gone
Remove and forbid use of com.google.common.hash.*
2 parents 96206df + 67d1c70 commit e5a10e9

File tree

8 files changed

+213
-58
lines changed

8 files changed

+213
-58
lines changed

‎core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.common.hash;
21+
22+
import org.elasticsearch.ElasticsearchException;
23+
24+
import java.security.MessageDigest;
25+
import java.security.NoSuchAlgorithmException;
26+
27+
public class MessageDigests {
28+
29+
private static final MessageDigest MD5_DIGEST;
30+
private static final MessageDigest SHA_1_DIGEST;
31+
private static final MessageDigest SHA_256_DIGEST;
32+
33+
static {
34+
try {
35+
MD5_DIGEST = MessageDigest.getInstance("MD5");
36+
SHA_1_DIGEST = MessageDigest.getInstance("SHA-1");
37+
SHA_256_DIGEST = MessageDigest.getInstance("SHA-256");
38+
} catch (NoSuchAlgorithmException e) {
39+
throw new ElasticsearchException("Unexpected exception creating MessageDigest instance", e);
40+
}
41+
}
42+
43+
public static MessageDigest md5() {
44+
return cloneAndReset(MD5_DIGEST);
45+
}
46+
47+
48+
public static MessageDigest sha1() {
49+
return cloneAndReset(SHA_1_DIGEST);
50+
}
51+
52+
public static MessageDigest sha256() {
53+
return cloneAndReset(SHA_256_DIGEST);
54+
}
55+
56+
private static MessageDigest cloneAndReset(MessageDigest messageDigest) {
57+
try {
58+
MessageDigest clone = (MessageDigest) messageDigest.clone();
59+
clone.reset();
60+
return clone;
61+
} catch (CloneNotSupportedException e) {
62+
throw new ElasticsearchException("Unexpected exception cloning MessageDigest instance", e);
63+
}
64+
}
65+
66+
private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray();
67+
public static String toHexString(byte[] bytes) {
68+
if (bytes == null) {
69+
throw new NullPointerException("bytes");
70+
}
71+
StringBuilder sb = new StringBuilder(2 * bytes.length);
72+
73+
for (int i = 0; i < bytes.length; i++) {
74+
byte b = bytes[i];
75+
sb.append(HEX_DIGITS[b >> 4 & 0xf]).append(HEX_DIGITS[b & 0xf]);
76+
}
77+
78+
return sb.toString();
79+
}
80+
}

‎core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,22 @@
1919

2020
package org.elasticsearch.common.http.client;
2121

22-
import java.nio.charset.StandardCharsets;
23-
import com.google.common.hash.Hashing;
2422
import org.apache.lucene.util.IOUtils;
25-
import org.elasticsearch.*;
23+
import org.elasticsearch.Build;
24+
import org.elasticsearch.ElasticsearchCorruptionException;
25+
import org.elasticsearch.ElasticsearchTimeoutException;
26+
import org.elasticsearch.Version;
2627
import org.elasticsearch.common.Base64;
2728
import org.elasticsearch.common.Nullable;
2829
import org.elasticsearch.common.Strings;
30+
import org.elasticsearch.common.hash.MessageDigests;
2931
import org.elasticsearch.common.unit.TimeValue;
3032

3133
import java.io.*;
3234
import java.net.HttpURLConnection;
3335
import java.net.URL;
3436
import java.net.URLConnection;
37+
import java.nio.charset.StandardCharsets;
3538
import java.nio.file.Files;
3639
import java.nio.file.NoSuchFileException;
3740
import java.nio.file.Path;
@@ -96,7 +99,7 @@ public interface Checksummer {
9699
public static Checksummer SHA1_CHECKSUM = new Checksummer() {
97100
@Override
98101
public String checksum(byte[] filebytes) {
99-
return Hashing.sha1().hashBytes(filebytes).toString();
102+
return MessageDigests.toHexString(MessageDigests.sha1().digest(filebytes));
100103
}
101104

102105
@Override
@@ -109,7 +112,7 @@ public String name() {
109112
public static Checksummer MD5_CHECKSUM = new Checksummer() {
110113
@Override
111114
public String checksum(byte[] filebytes) {
112-
return Hashing.md5().hashBytes(filebytes).toString();
115+
return MessageDigests.toHexString(MessageDigests.md5().digest(filebytes));
113116
}
114117

115118
@Override

‎core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,24 @@
1919

2020
package org.elasticsearch.cluster.routing.operation.hash.murmur3;
2121

22-
import com.carrotsearch.randomizedtesting.generators.RandomInts;
23-
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
24-
import com.google.common.hash.HashFunction;
25-
import com.google.common.hash.Hashing;
2622
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
2723
import org.elasticsearch.test.ESTestCase;
2824

2925
public class Murmur3HashFunctionTests extends ESTestCase {
3026

31-
public void test() {
32-
// Make sure that we agree with guava
33-
Murmur3HashFunction murmur3 = new Murmur3HashFunction();
34-
HashFunction guavaMurmur3 = Hashing.murmur3_32();
35-
for (int i = 0; i < 100; ++i) {
36-
final String id = RandomStrings.randomRealisticUnicodeOfCodepointLength(getRandom(), RandomInts.randomIntBetween(getRandom(), 1, 20));
37-
//final String id = "0";
38-
final int hash1 = guavaMurmur3.newHasher().putUnencodedChars(id).hash().asInt();
39-
final int hash2 = murmur3.hash(id);
40-
assertEquals(hash1, hash2);
41-
}
27+
private static Murmur3HashFunction HASH = new Murmur3HashFunction();
28+
29+
public void testKnownValues() {
30+
assertHash(0x5a0cb7c3, "hell");
31+
assertHash(0xd7c31989, "hello");
32+
assertHash(0x22ab2984, "hello w");
33+
assertHash(0xdf0ca123, "hello wo");
34+
assertHash(0xe7744d61, "hello wor");
35+
assertHash(0xe07db09c, "The quick brown fox jumps over the lazy dog");
36+
assertHash(0x4e63d2ad, "The quick brown fox jumps over the lazy cog");
4237
}
4338

39+
private static void assertHash(int expected, String stringInput) {
40+
assertEquals(expected, HASH.hash(stringInput));
41+
}
4442
}

‎core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.common.hash;
21+
22+
import org.elasticsearch.test.ESTestCase;
23+
import org.junit.Test;
24+
25+
import java.math.BigInteger;
26+
import java.nio.charset.StandardCharsets;
27+
import java.security.MessageDigest;
28+
29+
import static org.junit.Assert.*;
30+
31+
public class MessageDigestsTests extends ESTestCase {
32+
private void assertHash(String expected, String test, MessageDigest messageDigest) {
33+
String actual = MessageDigests.toHexString(messageDigest.digest(test.getBytes(StandardCharsets.UTF_8)));
34+
assertEquals(expected, actual);
35+
}
36+
37+
@Test
38+
public void testMd5() throws Exception {
39+
assertHash("d41d8cd98f00b204e9800998ecf8427e", "", MessageDigests.md5());
40+
assertHash("900150983cd24fb0d6963f7d28e17f72", "abc", MessageDigests.md5());
41+
assertHash("8215ef0796a20bcaaae116d3876c664a", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.md5());
42+
assertHash("7707d6ae4e027c70eea2a935c2296f21", new String(new char[1000000]).replace("\0", "a"), MessageDigests.md5());
43+
assertHash("9e107d9d372bb6826bd81d3542a419d6", "The quick brown fox jumps over the lazy dog", MessageDigests.md5());
44+
assertHash("1055d3e698d289f2af8663725127bd4b", "The quick brown fox jumps over the lazy cog", MessageDigests.md5());
45+
}
46+
47+
@Test
48+
public void testSha1() throws Exception {
49+
assertHash("da39a3ee5e6b4b0d3255bfef95601890afd80709", "", MessageDigests.sha1());
50+
assertHash("a9993e364706816aba3e25717850c26c9cd0d89d", "abc", MessageDigests.sha1());
51+
assertHash("84983e441c3bd26ebaae4aa1f95129e5e54670f1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha1());
52+
assertHash("34aa973cd4c4daa4f61eeb2bdbad27316534016f", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha1());
53+
assertHash("2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", "The quick brown fox jumps over the lazy dog", MessageDigests.sha1());
54+
assertHash("de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3", "The quick brown fox jumps over the lazy cog", MessageDigests.sha1());
55+
}
56+
57+
@Test
58+
public void testSha256() throws Exception {
59+
assertHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "", MessageDigests.sha256());
60+
assertHash("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc", MessageDigests.sha256());
61+
assertHash("248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha256());
62+
assertHash("cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha256());
63+
assertHash("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", "The quick brown fox jumps over the lazy dog", MessageDigests.sha256());
64+
assertHash("e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be", "The quick brown fox jumps over the lazy cog", MessageDigests.sha256());
65+
}
66+
67+
@Test
68+
public void testToHexString() throws Exception {
69+
for (int i = 0; i < 1024; i++) {
70+
BigInteger expected = BigInteger.probablePrime(256, random());
71+
byte[] bytes = expected.toByteArray();
72+
String hex = MessageDigests.toHexString(bytes);
73+
String zeros = new String(new char[bytes.length * 2]).replace("\0", "0");
74+
String expectedAsString = expected.toString(16);
75+
String expectedHex = zeros.substring(expectedAsString.length()) + expectedAsString;
76+
assertEquals(expectedHex, hex);
77+
BigInteger actual = new BigInteger(hex, 16);
78+
assertEquals(expected, actual);
79+
}
80+
}
81+
}

‎core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,34 @@
1919

2020
package org.elasticsearch.common.hashing;
2121

22-
import com.google.common.hash.HashCode;
23-
import com.google.common.hash.Hashing;
2422
import org.elasticsearch.common.hash.MurmurHash3;
2523
import org.elasticsearch.test.ESTestCase;
2624

27-
import java.nio.ByteBuffer;
28-
import java.nio.ByteOrder;
29-
import java.nio.LongBuffer;
25+
import java.io.UnsupportedEncodingException;
26+
import java.nio.charset.StandardCharsets;
3027

3128
public class MurmurHash3Tests extends ESTestCase {
32-
33-
public void testHash128() {
34-
final int iters = scaledRandomIntBetween(100, 5000);
35-
for (int i = 0; i < iters; ++i) {
36-
final int seed = randomInt();
37-
final int offset = randomInt(20);
38-
final int len = randomInt(randomBoolean() ? 20 : 200);
39-
final byte[] bytes = new byte[len + offset + randomInt(3)];
40-
getRandom().nextBytes(bytes);
41-
HashCode h1 = Hashing.murmur3_128(seed).hashBytes(bytes, offset, len);
42-
MurmurHash3.Hash128 h2 = MurmurHash3.hash128(bytes, offset, len, seed, new MurmurHash3.Hash128());
43-
assertEquals(h1, h2);
44-
}
29+
public void testKnownValues() throws UnsupportedEncodingException {
30+
assertHash(0x629942693e10f867L, 0x92db0b82baeb5347L, "hell", 0);
31+
assertHash(0xa78ddff5adae8d10L, 0x128900ef20900135L, "hello", 1);
32+
assertHash(0x8a486b23f422e826L, 0xf962a2c58947765fL, "hello ", 2);
33+
assertHash(0x2ea59f466f6bed8cL, 0xc610990acc428a17L, "hello w", 3);
34+
assertHash(0x79f6305a386c572cL, 0x46305aed3483b94eL, "hello wo", 4);
35+
assertHash(0xc2219d213ec1f1b5L, 0xa1d8e2e0a52785bdL, "hello wor", 5);
36+
assertHash(0xe34bbc7bbc071b6cL, 0x7a433ca9c49a9347L, "The quick brown fox jumps over the lazy dog", 0);
37+
assertHash(0x658ca970ff85269aL, 0x43fee3eaa68e5c3eL, "The quick brown fox jumps over the lazy cog", 0);
4538
}
4639

47-
private void assertEquals(HashCode h1, MurmurHash3.Hash128 h2) {
48-
final LongBuffer longs = ByteBuffer.wrap(h1.asBytes()).order(ByteOrder.LITTLE_ENDIAN).asLongBuffer();
49-
assertEquals(2, longs.limit());
50-
assertEquals(h1.asLong(), h2.h1);
51-
assertEquals(longs.get(), h2.h1);
52-
assertEquals(longs.get(), h2.h2);
40+
private static void assertHash(long lower, long upper, String inputString, long seed) {
41+
byte[] bytes = inputString.getBytes(StandardCharsets.UTF_8);
42+
MurmurHash3.Hash128 expected = new MurmurHash3.Hash128();
43+
expected.h1 = lower;
44+
expected.h2 = upper;
45+
assertHash(expected, MurmurHash3.hash128(bytes, 0, bytes.length, seed, new MurmurHash3.Hash128()));
5346
}
5447

48+
private static void assertHash(MurmurHash3.Hash128 expected, MurmurHash3.Hash128 actual) {
49+
assertEquals(expected.h1, actual.h1);
50+
assertEquals(expected.h2, actual.h2);
51+
}
5552
}

‎core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,14 @@
1818
*/
1919
package org.elasticsearch.plugins;
2020

21-
import java.nio.charset.StandardCharsets;
22-
import com.google.common.hash.Hashing;
23-
2421
import org.apache.http.impl.client.HttpClients;
2522
import org.apache.lucene.util.LuceneTestCase;
2623
import org.elasticsearch.Version;
2724
import org.elasticsearch.common.Base64;
2825
import org.elasticsearch.common.cli.CliTool;
2926
import org.elasticsearch.common.cli.CliTool.ExitStatus;
3027
import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal;
28+
import org.elasticsearch.common.hash.MessageDigests;
3129
import org.elasticsearch.common.settings.Settings;
3230
import org.elasticsearch.env.Environment;
3331
import org.elasticsearch.node.internal.InternalSettingsPreparer;
@@ -46,16 +44,15 @@
4644
import org.jboss.netty.handler.ssl.util.SelfSignedCertificate;
4745
import org.junit.After;
4846
import org.junit.Before;
49-
import org.junit.Test;
5047

5148
import javax.net.ssl.HttpsURLConnection;
5249
import javax.net.ssl.SSLContext;
5350
import javax.net.ssl.SSLSocketFactory;
54-
5551
import java.io.BufferedWriter;
5652
import java.io.IOException;
5753
import java.net.InetAddress;
5854
import java.net.InetSocketAddress;
55+
import java.nio.charset.StandardCharsets;
5956
import java.nio.file.FileVisitResult;
6057
import java.nio.file.Files;
6158
import java.nio.file.Path;
@@ -109,7 +106,7 @@ public void clearPathHome() {
109106
}
110107

111108
private void writeSha1(Path file, boolean corrupt) throws IOException {
112-
String sha1Hex = Hashing.sha1().hashBytes(Files.readAllBytes(file)).toString();
109+
String sha1Hex = MessageDigests.toHexString(MessageDigests.sha1().digest(Files.readAllBytes(file)));
113110
try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".sha1"), StandardCharsets.UTF_8)) {
114111
out.write(sha1Hex);
115112
if (corrupt) {
@@ -119,7 +116,7 @@ private void writeSha1(Path file, boolean corrupt) throws IOException {
119116
}
120117

121118
private void writeMd5(Path file, boolean corrupt) throws IOException {
122-
String md5Hex = Hashing.md5().hashBytes(Files.readAllBytes(file)).toString();
119+
String md5Hex = MessageDigests.toHexString(MessageDigests.md5().digest(Files.readAllBytes(file)));
123120
try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".md5"), StandardCharsets.UTF_8)) {
124121
out.write(md5Hex);
125122
if (corrupt) {

‎dev-tools/src/main/resources/forbidden/all-signatures.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ com.google.common.primitives.Ints
130130
com.google.common.collect.ImmutableSet
131131
com.google.common.collect.ImmutableSet$Builder
132132
com.google.common.io.Resources
133+
com.google.common.hash.HashCode
134+
com.google.common.hash.HashFunction
135+
com.google.common.hash.Hashing
133136

134137
@defaultMessage Do not violate java's access system
135138
java.lang.reflect.AccessibleObject#setAccessible(boolean)

0 commit comments

Comments
 (0)