From bc6eff20a23cc931be13676681763cad483f4380 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:14:09 +0000 Subject: [PATCH 1/2] Fix client-v2: do not declare a content encoding on a multipart request The multipart body used to send statement parameters is always attached uncompressed, but the request still carried Content-Encoding: lz4 when both client request compression and HTTP compression were enabled. ClickHouse 26.8+ honours that header for multipart requests and fails the query with LZ4_DECODER_FAILED. The header is now removed for multipart requests, next to the decompress query parameter that was already dropped for them. Fixes: https://github.com/ClickHouse/clickhouse-java/issues/3075 --- CHANGELOG.md | 7 ++ .../api/internal/HttpAPIClientHelper.java | 16 ++- .../api/internal/HttpAPIClientHelperTest.java | 100 ++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b264b9415..fc415f430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,6 +153,13 @@ ### Bug Fixes +- **[client-v2]** Fixed a query with statement parameters sent in the request body + (`client.http.use_form_request_for_query=true`) failing with `LZ4 decompression failed ... (LZ4_DECODER_FAILED)` + when client request compression and HTTP compression were both enabled. The multipart body is always sent + uncompressed, but the request still declared `Content-Encoding: lz4`; ClickHouse `26.8+` honours that header for + multipart requests and tried to decompress a plain body. The header is now omitted for multipart requests, like + the `decompress` query parameter already was. Response compression (`Accept-Encoding`, + `enable_http_compression`) is unchanged. (https://github.com/ClickHouse/clickhouse-java/issues/3075) - **[jdbc-v2, client-v2]** Fixes issue with `FORMAT` in query unable to override format set by client when used with ClickHouse 26.8+. Default format is `RowBinaryWithNamesAndTypes` set at client level. For JDBC, recommend using `format=JSONEachRow` to query JSON. Setting `format=` (empty or `null`) omits the format request header so explicit diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 036ed48d2..28ab8986b 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -570,10 +570,10 @@ private URI createRequestURI(Endpoint server, Map requestConfig, return uri; } - private HttpPost createPostRequest(URI uri, Map requestConfig) { + private HttpPost createPostRequest(URI uri, Map requestConfig, boolean isMultipartRequest) { HttpPost req = new HttpPost(uri); // req.setVersion(new ProtocolVersion("HTTP", 1, 0)); // to disable chunk transfer encoding - addHeaders(req, requestConfig); + addHeaders(req, requestConfig, isMultipartRequest); return req; } @@ -620,7 +620,7 @@ public TransportRequest createRequest(Endpoint server, Map reque // create configuration dependent objects final URI uri = createRequestURI(server, requestConfig, useMultipart); - final HttpPost req = createPostRequest(uri, requestConfig); + final HttpPost req = createPostRequest(uri, requestConfig, useMultipart); final HttpEntity httpEntity; if (useMultipart) { @@ -829,7 +829,7 @@ private TransportResponse doExecuteRequest(TransportRequest transportRequest, Sp public TransportRequest createRequest(Endpoint server, Map requestConfig, IOCallback writeCallback) { final URI uri = createRequestURI(server, requestConfig, false); - final HttpPost req = createPostRequest(uri, requestConfig); + final HttpPost req = createPostRequest(uri, requestConfig, false); try { String contentEncoding = req.containsHeader(HttpHeaders.CONTENT_ENCODING) ? req.getHeader(HttpHeaders.CONTENT_ENCODING).getValue() : null; req.setEntity(wrapRequestEntity( @@ -869,7 +869,7 @@ private void logServerErrorResponse(HttpPost req, ClassicHttpResponse httpRespon private static final ContentType CONTENT_TYPE = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), "UTF-8"); - private void addHeaders(HttpPost req, Map requestConfig) { + private void addHeaders(HttpPost req, Map requestConfig, boolean isMultipartRequest) { setHeader(req, HttpHeaders.CONTENT_TYPE, CONTENT_TYPE.getMimeType()); if (requestConfig.containsKey(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())) { Object formatObj = requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()); @@ -955,6 +955,12 @@ private void addHeaders(HttpPost req, Map requestConfig) { } // Special cases + if (isMultipartRequest) { + // a multipart body is sent as-is (see createRequest), so any content encoding would make the server + // fail to decompress the request + req.removeHeaders(HttpHeaders.CONTENT_ENCODING); + } + if (req.containsHeader(HttpHeaders.AUTHORIZATION) && (req.containsHeader(ClickHouseHttpProto.HEADER_DB_USER) || req.containsHeader(ClickHouseHttpProto.HEADER_DB_PASSWORD))) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index 16841dc6e..3c9a656a5 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -13,7 +13,9 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.message.BasicHeader; import org.mockito.ArgumentCaptor; import org.mockito.MockedConstruction; @@ -327,6 +329,104 @@ public void testShouldRetryUsesServerExceptionFromCause(Throwable ex, boolean ex assertEquals(helper.shouldRetry(ex, new HashMap<>()), expectedRetry); } + /** + * A multipart body (statement parameters sent as form data) is never compressed, so the request must not + * declare a content encoding - the server would try to decompress the plain body and fail with + * LZ4_DECODER_FAILED. A request that is not multipart, and response compression, keep their signalling. + */ + @DataProvider(name = "requestCompressionSignalling") + public static Object[][] requestCompressionSignalling() { + return new Object[][] { + // clientCompression, useHttpCompression, sendParamsInBody, withParams, + // contentEncoding, acceptEncoding, decompressParam + {true, true, true, true, null, "lz4", false}, + {true, true, true, false, "lz4", "lz4", false}, // no parameters -> not a multipart request + {true, true, false, true, "lz4", "lz4", false}, + {false, true, true, true, null, "lz4", false}, + {true, false, true, true, null, null, false}, + {true, false, false, true, null, null, true}, + }; + } + + @Test(dataProvider = "requestCompressionSignalling") + public void testRequestCompressionSignalling(boolean clientCompression, boolean useHttpCompression, + boolean sendParamsInBody, boolean withParams, + String expectedContentEncoding, String expectedAcceptEncoding, + boolean expectDecompressParam) { + Map reqConfig = compressionConfig(clientCompression, useHttpCompression, sendParamsInBody); + if (withParams) { + reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1")); + } + + HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig, + "SELECT {p1:Int32}").getDelegate(); + + String setup = "clientCompression=" + clientCompression + ", useHttpCompression=" + useHttpCompression + + ", sendParamsInBody=" + sendParamsInBody + ", withParams=" + withParams; + assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), expectedContentEncoding, + "unexpected " + HttpHeaders.CONTENT_ENCODING + " for " + setup); + assertEquals(req.getEntity().getContentEncoding(), expectedContentEncoding, + "the request body entity must declare the same encoding as the request for " + setup); + assertEquals(headerValue(req, HttpHeaders.ACCEPT_ENCODING), expectedAcceptEncoding, + "response compression signalling must not depend on the request body form"); + + String query = req.getRequestUri(); + assertEquals(query.contains(ClickHouseHttpProto.QPARAM_DECOMPRESS + "=1"), expectDecompressParam, + "unexpected " + ClickHouseHttpProto.QPARAM_DECOMPRESS + " parameter in " + query); + assertEquals(query.contains(ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + "=1"), useHttpCompression, + "unexpected " + ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + " parameter in " + query); + } + + /** + * A content encoding set by the application through {@code http_header_*} cannot make the plain multipart + * body compressed either, so it must not reach the server. + */ + @Test + public void testCustomContentEncodingHeaderRemovedForMultipartRequest() { + Map reqConfig = compressionConfig(false, false, true); + reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1")); + reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + HttpHeaders.CONTENT_ENCODING, "lz4"); + + HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig, + "SELECT {p1:Int32}").getDelegate(); + + assertNull(headerValue(req, HttpHeaders.CONTENT_ENCODING), + "a custom " + HttpHeaders.CONTENT_ENCODING + " must be removed from a multipart request"); + } + + /** + * Data is streamed into the request body, so an insert is never a multipart request and keeps compressing + * its body even when the client is configured to send statement parameters in the body. + */ + @Test + public void testDataRequestKeepsContentEncodingWhenParamsInBodyEnabled() { + Map reqConfig = compressionConfig(true, true, true); + + HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig, + out -> out.write(1)).getDelegate(); + + assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), "lz4", + "an insert body is compressed, so the request must declare the content encoding"); + } + + private static HttpAPIClientHelper newHelper() { + return HttpAPIClientHelperFactory.newHelper(new HashMap<>(), LZ4Factory.fastestInstance()); + } + + private static Map compressionConfig(boolean clientCompression, boolean useHttpCompression, + boolean sendParamsInBody) { + Map reqConfig = new HashMap<>(); + reqConfig.put(ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey(), clientCompression); + reqConfig.put(ClientConfigProperties.USE_HTTP_COMPRESSION.getKey(), useHttpCompression); + reqConfig.put(ClientConfigProperties.HTTP_SEND_PARAMS_IN_BODY.getKey(), sendParamsInBody); + return reqConfig; + } + + private static String headerValue(HttpPost req, String name) { + Header header = req.getFirstHeader(name); + return header == null ? null : header.getValue(); + } + /** * A server error is logged at WARN only for an unknown status code (the switch's default branch). Known * error paths emit no server-error WARN: readError surfaces an exception-code error, a mapped code (502) From f7e53ef084bb753a9113af960bbca005b58022ad Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:29:27 +0000 Subject: [PATCH 2/2] Remove the multipart content encoding at the request build site Keeps addHeaders() untouched: threading a multipart flag through it changed its declaration line, which reported its pre-existing cognitive complexity (sonar java:S3776) as a new-code issue and failed the quality gate. The header is now removed in createRequest(), still after addHeaders() ran, so an encoding set by the application with http_header_* is dropped too. Also pins the header spelling and the non-multipart contrast case. --- .../api/internal/HttpAPIClientHelper.java | 21 ++++++------- .../api/internal/HttpAPIClientHelperTest.java | 31 ++++++++++++++++--- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 28ab8986b..9dab47a39 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -570,10 +570,10 @@ private URI createRequestURI(Endpoint server, Map requestConfig, return uri; } - private HttpPost createPostRequest(URI uri, Map requestConfig, boolean isMultipartRequest) { + private HttpPost createPostRequest(URI uri, Map requestConfig) { HttpPost req = new HttpPost(uri); // req.setVersion(new ProtocolVersion("HTTP", 1, 0)); // to disable chunk transfer encoding - addHeaders(req, requestConfig, isMultipartRequest); + addHeaders(req, requestConfig); return req; } @@ -620,10 +620,15 @@ public TransportRequest createRequest(Endpoint server, Map reque // create configuration dependent objects final URI uri = createRequestURI(server, requestConfig, useMultipart); - final HttpPost req = createPostRequest(uri, requestConfig, useMultipart); + final HttpPost req = createPostRequest(uri, requestConfig); final HttpEntity httpEntity; if (useMultipart) { + // a multipart body is always sent as-is, so the request must not declare a content encoding - the + // server would fail to decompress the plain body. Removed after addHeaders() to also drop an + // encoding set by the application with `http_header_*`. + req.removeHeaders(HttpHeaders.CONTENT_ENCODING); + MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); addStatementParams(requestConfig, multipartEntityBuilder::addTextBody); multipartEntityBuilder.addTextBody(ClickHouseHttpProto.QPARAM_QUERY_STMT, body); @@ -829,7 +834,7 @@ private TransportResponse doExecuteRequest(TransportRequest transportRequest, Sp public TransportRequest createRequest(Endpoint server, Map requestConfig, IOCallback writeCallback) { final URI uri = createRequestURI(server, requestConfig, false); - final HttpPost req = createPostRequest(uri, requestConfig, false); + final HttpPost req = createPostRequest(uri, requestConfig); try { String contentEncoding = req.containsHeader(HttpHeaders.CONTENT_ENCODING) ? req.getHeader(HttpHeaders.CONTENT_ENCODING).getValue() : null; req.setEntity(wrapRequestEntity( @@ -869,7 +874,7 @@ private void logServerErrorResponse(HttpPost req, ClassicHttpResponse httpRespon private static final ContentType CONTENT_TYPE = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), "UTF-8"); - private void addHeaders(HttpPost req, Map requestConfig, boolean isMultipartRequest) { + private void addHeaders(HttpPost req, Map requestConfig) { setHeader(req, HttpHeaders.CONTENT_TYPE, CONTENT_TYPE.getMimeType()); if (requestConfig.containsKey(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())) { Object formatObj = requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()); @@ -955,12 +960,6 @@ private void addHeaders(HttpPost req, Map requestConfig, boolean } // Special cases - if (isMultipartRequest) { - // a multipart body is sent as-is (see createRequest), so any content encoding would make the server - // fail to decompress the request - req.removeHeaders(HttpHeaders.CONTENT_ENCODING); - } - if (req.containsHeader(HttpHeaders.AUTHORIZATION) && (req.containsHeader(ClickHouseHttpProto.HEADER_DB_USER) || req.containsHeader(ClickHouseHttpProto.HEADER_DB_PASSWORD))) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index 3c9a656a5..0c7fa375a 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -377,21 +377,42 @@ public void testRequestCompressionSignalling(boolean clientCompression, boolean "unexpected " + ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + " parameter in " + query); } + @DataProvider(name = "contentEncodingHeaderNames") + public static Object[][] contentEncodingHeaderNames() { + return new Object[][] {{HttpHeaders.CONTENT_ENCODING}, {"content-encoding"}}; + } + /** * A content encoding set by the application through {@code http_header_*} cannot make the plain multipart - * body compressed either, so it must not reach the server. + * body compressed either, so it must not reach the server, whatever the header is spelled like. */ - @Test - public void testCustomContentEncodingHeaderRemovedForMultipartRequest() { + @Test(dataProvider = "contentEncodingHeaderNames") + public void testCustomContentEncodingHeaderRemovedForMultipartRequest(String headerName) { Map reqConfig = compressionConfig(false, false, true); reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1")); - reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + HttpHeaders.CONTENT_ENCODING, "lz4"); + reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + headerName, "lz4"); HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig, "SELECT {p1:Int32}").getDelegate(); assertNull(headerValue(req, HttpHeaders.CONTENT_ENCODING), - "a custom " + HttpHeaders.CONTENT_ENCODING + " must be removed from a multipart request"); + "a custom " + headerName + " must be removed from a multipart request"); + } + + /** + * A request that is not multipart is unaffected: a content encoding set by the application through + * {@code http_header_*} still reaches the server. + */ + @Test(dataProvider = "contentEncodingHeaderNames") + public void testCustomContentEncodingHeaderKeptForRequestWithoutParams(String headerName) { + Map reqConfig = compressionConfig(false, false, true); + reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + headerName, "lz4"); + + HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig, + "SELECT 1").getDelegate(); + + assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), "lz4", + "a custom " + headerName + " must be kept on a request that is not multipart"); } /**