diff --git a/LICENSES/LICENSE.zstd-jni.txt b/LICENSES/LICENSE.zstd-jni.txt new file mode 100644 index 000000000..66abb8ae7 --- /dev/null +++ b/LICENSES/LICENSE.zstd-jni.txt @@ -0,0 +1,26 @@ +Zstd-jni: JNI bindings to Zstd Library + +Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. + +BSD License + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java index db07a6323..48662636b 100755 --- a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java +++ b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java @@ -18,6 +18,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.handler.codec.compression.Brotli; +import io.netty.handler.codec.compression.Zstd; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.DefaultHttpRequest; import io.netty.handler.codec.http.HttpHeaderValues; @@ -67,6 +68,7 @@ import static org.asynchttpclient.util.HttpUtils.ACCEPT_ALL_HEADER_VALUE; import static org.asynchttpclient.util.HttpUtils.GZIP_DEFLATE; import static org.asynchttpclient.util.HttpUtils.filterOutBrotliFromAcceptEncoding; +import static org.asynchttpclient.util.HttpUtils.filterOutZstdFromAcceptEncoding; import static org.asynchttpclient.util.HttpUtils.hostHeader; import static org.asynchttpclient.util.HttpUtils.originHeader; import static org.asynchttpclient.util.HttpUtils.urlEncodeFormParams; @@ -182,6 +184,11 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque // For manual decompression by user, any encoding may suite, so leave untouched headers.set(ACCEPT_ENCODING, filterOutBrotliFromAcceptEncoding(userDefinedAcceptEncoding)); } + if (!Zstd.isAvailable()) { + // zstd is not available. + // For manual decompression by user, any encoding may suit, so leave untouched + headers.set(ACCEPT_ENCODING, filterOutZstdFromAcceptEncoding(userDefinedAcceptEncoding)); + } } } else if (config.isCompressionEnforced()) { // Add Accept Encoding header if compression is enforced @@ -189,6 +196,9 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque if (Brotli.isAvailable()) { headers.add(ACCEPT_ENCODING, HttpHeaderValues.BR); } + if (Zstd.isAvailable()) { + headers.add(ACCEPT_ENCODING, HttpHeaderValues.ZSTD); + } } } diff --git a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java index f62e2f235..3cca41e61 100644 --- a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java +++ b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java @@ -40,6 +40,7 @@ public final class HttpUtils { private static final String CONTENT_TYPE_CHARSET_ATTRIBUTE = "charset="; private static final String CONTENT_TYPE_BOUNDARY_ATTRIBUTE = "boundary="; private static final String BROTLY_ACCEPT_ENCODING_SUFFIX = ", br"; + private static final String ZSTD_ACCEPT_ENCODING_SUFFIX = ", zstd"; private HttpUtils() { // Prevent outside initialization @@ -173,4 +174,12 @@ public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncodi } return acceptEncoding; } + + public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding) { + // we don't support zstd ATM + if (acceptEncoding.endsWith(ZSTD_ACCEPT_ENCODING_SUFFIX)) { + return acceptEncoding.subSequence(0, acceptEncoding.length() - ZSTD_ACCEPT_ENCODING_SUFFIX.length()); + } + return acceptEncoding; + } } diff --git a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java new file mode 100644 index 000000000..238bb7206 --- /dev/null +++ b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2015-2024 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.asynchttpclient; + +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; +import com.aayushatharva.brotli4j.encoder.Encoder; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import com.github.luben.zstd.Zstd; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AutomaticDecompressionTest { + private static final String UNCOMPRESSED_PAYLOAD = "a".repeat(500); + + private static HttpServer HTTP_SERVER; + + private static AsyncHttpClient createClient() { + AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder() + .setEnableAutomaticDecompression(true) + .setCompressionEnforced(true) + .build(); + return new DefaultAsyncHttpClient(config); + } + + @BeforeAll + static void setupServer() throws Exception { + HTTP_SERVER = HttpServer.create(new InetSocketAddress(0), 0); + + HTTP_SERVER.createContext("/br").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + validateAcceptEncodingHeader(exchange); + exchange.getResponseHeaders().set("Content-Encoding", "br"); + exchange.sendResponseHeaders(200, 0); + OutputStream out = exchange.getResponseBody(); + Encoder.Parameters params = new Encoder.Parameters(); + BrotliOutputStream brotliOutputStream = new BrotliOutputStream(out, params); + brotliOutputStream.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); + brotliOutputStream.flush(); + brotliOutputStream.close(); + } + }); + + HTTP_SERVER.createContext("/zstd").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + validateAcceptEncodingHeader(exchange); + exchange.getResponseHeaders().set("Content-Encoding", "zstd"); + byte[] compressedData = new byte[UNCOMPRESSED_PAYLOAD.length()]; + long n = Zstd.compress(compressedData, UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8), 2, true); + exchange.sendResponseHeaders(200, n); + OutputStream out = exchange.getResponseBody(); + out.write(compressedData, 0, (int) n); + out.flush(); + out.close(); + } + }); + + HTTP_SERVER.createContext("/gzip").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + validateAcceptEncodingHeader(exchange); + exchange.getResponseHeaders().set("Content-Encoding", "gzip"); + exchange.sendResponseHeaders(200, 0); + OutputStream out = exchange.getResponseBody(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); + gzip.flush(); + gzip.close(); + } + }); + + HTTP_SERVER.start(); + } + + private static void validateAcceptEncodingHeader(HttpExchange exchange) { + Headers requestHeaders = exchange.getRequestHeaders(); + List acceptEncodingList = requestHeaders.get("Accept-Encoding") + .stream() + .flatMap(x -> Arrays.asList(x.split(",")).stream()) + .collect(Collectors.toList()); + assertEquals(List.of("gzip", "deflate", "br", "zstd"), acceptEncodingList); + } + + @AfterAll + static void stopServer() { + if (HTTP_SERVER != null) { + HTTP_SERVER.stop(0); + } + } + + @Test + void zstd() throws Throwable { + io.netty.handler.codec.compression.Zstd.ensureAvailability(); + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/zstd") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + @Test + void brotli() throws Throwable { + io.netty.handler.codec.compression.Brotli.ensureAvailability(); + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/br") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + @Test + void gzip() throws Throwable { + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/gzip") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + +} diff --git a/client/src/test/java/org/asynchttpclient/netty/NettyTest.java b/client/src/test/java/org/asynchttpclient/netty/NettyTest.java index 9a0293be3..7878766f1 100644 --- a/client/src/test/java/org/asynchttpclient/netty/NettyTest.java +++ b/client/src/test/java/org/asynchttpclient/netty/NettyTest.java @@ -3,6 +3,7 @@ import io.netty.channel.epoll.Epoll; import io.netty.channel.kqueue.KQueue; import io.netty.handler.codec.compression.Brotli; +import io.netty.handler.codec.compression.Zstd; import io.netty.incubator.channel.uring.IOUring; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; @@ -40,4 +41,16 @@ public void brotliIsAvailableOnLinux() { public void brotliIsAvailableOnMac() { assertTrue(Brotli.isAvailable()); } + + @Test + @EnabledOnOs(value = OS.LINUX) + public void zstdIsAvailableOnLinux() { + assertTrue(Zstd.isAvailable()); + } + + @Test + @EnabledOnOs(value = OS.MAC) + public void zstdIsAvailableOnMac() { + assertTrue(Zstd.isAvailable()); + } } diff --git a/pom.xml b/pom.xml index 42fcee0f7..1a6de507d 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ 0.0.25.Final 1.16.0 2.0.13 + 1.5.6-3 2.0.1 1.4.11 24.0.1 @@ -224,6 +225,13 @@ true + + com.github.luben + zstd-jni + ${zstd-jni.version} + true + + com.aayushatharva.brotli4j brotli4j