Fix client-v2: do not declare a content encoding on a multipart request

CHANGELOG.md

@@ -153,6 +153,13 @@
 
 ### Bug Fixes 
 
+- **[client-v2]** Fixed a query with statement parameters sent in the request body
+  (`client.http.use_form_request_for_query=true`) failing with `LZ4 decompression failed ... (LZ4_DECODER_FAILED)`
+  when client request compression and HTTP compression were both enabled. The multipart body is always sent
+  uncompressed, but the request still declared `Content-Encoding: lz4`; ClickHouse `26.8+` honours that header for
+  multipart requests and tried to decompress a plain body. The header is now omitted for multipart requests, like
+  the `decompress` query parameter already was. Response compression (`Accept-Encoding`,
+  `enable_http_compression`) is unchanged. (https://github.com/ClickHouse/clickhouse-java/issues/3075)
 - **[jdbc-v2, client-v2]** Fixes issue with `FORMAT` in query unable to override format set by client when used with
   ClickHouse 26.8+. Default format is `RowBinaryWithNamesAndTypes` set at client level. For JDBC, recommend using
   `format=JSONEachRow` to query JSON. Setting `format=` (empty or `null`) omits the format request header so explicit

client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java

@@ -624,6 +624,11 @@ public TransportRequest createRequest(Endpoint server, Map<String, Object> reque
 
         final HttpEntity httpEntity;
         if (useMultipart) {
+            // a multipart body is always sent as-is, so the request must not declare a content encoding - the
+            // server would fail to decompress the plain body. Removed after addHeaders() to also drop an
+            // encoding set by the application with `http_header_*`.
+            req.removeHeaders(HttpHeaders.CONTENT_ENCODING);
+
             MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
             addStatementParams(requestConfig, multipartEntityBuilder::addTextBody);
             multipartEntityBuilder.addTextBody(ClickHouseHttpProto.QPARAM_QUERY_STMT, body);

client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java

@@ -13,7 +13,9 @@
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
 import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHeaders;
 import org.apache.hc.core5.http.message.BasicHeader;
 import org.mockito.ArgumentCaptor;
 import org.mockito.MockedConstruction;
@@ -327,6 +329,125 @@ public void testShouldRetryUsesServerExceptionFromCause(Throwable ex, boolean ex
         assertEquals(helper.shouldRetry(ex, new HashMap<>()), expectedRetry);
     }
 
+    /**
+     * A multipart body (statement parameters sent as form data) is never compressed, so the request must not
+     * declare a content encoding - the server would try to decompress the plain body and fail with
+     * LZ4_DECODER_FAILED. A request that is not multipart, and response compression, keep their signalling.
+     */
+    @DataProvider(name = "requestCompressionSignalling")
+    public static Object[][] requestCompressionSignalling() {
+        return new Object[][] {
+                // clientCompression, useHttpCompression, sendParamsInBody, withParams,
+                //         contentEncoding, acceptEncoding, decompressParam
+                {true, true, true, true, null, "lz4", false},
+                {true, true, true, false, "lz4", "lz4", false}, // no parameters -> not a multipart request
+                {true, true, false, true, "lz4", "lz4", false},
+                {false, true, true, true, null, "lz4", false},
+                {true, false, true, true, null, null, false},
+                {true, false, false, true, null, null, true},
+        };
+    }
+
+    @Test(dataProvider = "requestCompressionSignalling")
+    public void testRequestCompressionSignalling(boolean clientCompression, boolean useHttpCompression,
+                                                 boolean sendParamsInBody, boolean withParams,
+                                                 String expectedContentEncoding, String expectedAcceptEncoding,
+                                                 boolean expectDecompressParam) {
+        Map<String, Object> reqConfig = compressionConfig(clientCompression, useHttpCompression, sendParamsInBody);
+        if (withParams) {
+            reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1"));
+        }
+
+        HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig,
+                "SELECT {p1:Int32}").getDelegate();
+
+        String setup = "clientCompression=" + clientCompression + ", useHttpCompression=" + useHttpCompression
+                + ", sendParamsInBody=" + sendParamsInBody + ", withParams=" + withParams;
+        assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), expectedContentEncoding,
+                "unexpected " + HttpHeaders.CONTENT_ENCODING + " for " + setup);
+        assertEquals(req.getEntity().getContentEncoding(), expectedContentEncoding,
+                "the request body entity must declare the same encoding as the request for " + setup);
+        assertEquals(headerValue(req, HttpHeaders.ACCEPT_ENCODING), expectedAcceptEncoding,
+                "response compression signalling must not depend on the request body form");
+
+        String query = req.getRequestUri();
+        assertEquals(query.contains(ClickHouseHttpProto.QPARAM_DECOMPRESS + "=1"), expectDecompressParam,
+                "unexpected " + ClickHouseHttpProto.QPARAM_DECOMPRESS + " parameter in " + query);
+        assertEquals(query.contains(ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + "=1"), useHttpCompression,
+                "unexpected " + ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + " parameter in " + query);
+    }
+
+    @DataProvider(name = "contentEncodingHeaderNames")
+    public static Object[][] contentEncodingHeaderNames() {
+        return new Object[][] {{HttpHeaders.CONTENT_ENCODING}, {"content-encoding"}};
+    }
+
+    /**
+     * A content encoding set by the application through {@code http_header_*} cannot make the plain multipart
+     * body compressed either, so it must not reach the server, whatever the header is spelled like.
+     */
+    @Test(dataProvider = "contentEncodingHeaderNames")
+    public void testCustomContentEncodingHeaderRemovedForMultipartRequest(String headerName) {
+        Map<String, Object> reqConfig = compressionConfig(false, false, true);
+        reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1"));
+        reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + headerName, "lz4");
+
+        HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig,
+                "SELECT {p1:Int32}").getDelegate();
+
+        assertNull(headerValue(req, HttpHeaders.CONTENT_ENCODING),
+                "a custom " + headerName + " must be removed from a multipart request");
+    }
+
+    /**
+     * A request that is not multipart is unaffected: a content encoding set by the application through
+     * {@code http_header_*} still reaches the server.
+     */
+    @Test(dataProvider = "contentEncodingHeaderNames")
+    public void testCustomContentEncodingHeaderKeptForRequestWithoutParams(String headerName) {
+        Map<String, Object> reqConfig = compressionConfig(false, false, true);
+        reqConfig.put(ClientConfigProperties.HTTP_HEADER_PREFIX + headerName, "lz4");
+
+        HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig,
+                "SELECT 1").getDelegate();
+
+        assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), "lz4",
+                "a custom " + headerName + " must be kept on a request that is not multipart");
+    }
+
+    /**
+     * Data is streamed into the request body, so an insert is never a multipart request and keeps compressing
+     * its body even when the client is configured to send statement parameters in the body.
+     */
+    @Test
+    public void testDataRequestKeepsContentEncodingWhenParamsInBodyEnabled() {
+        Map<String, Object> reqConfig = compressionConfig(true, true, true);
+
+        HttpPost req = newHelper().createRequest(new HttpEndpoint("localhost", 8123, false, "/"), reqConfig,
+                out -> out.write(1)).getDelegate();
+
+        assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), "lz4",
+                "an insert body is compressed, so the request must declare the content encoding");
+    }
+
+    private static HttpAPIClientHelper newHelper() {
+        return HttpAPIClientHelperFactory.newHelper(new HashMap<>(), LZ4Factory.fastestInstance());
+    }
+
+    private static Map<String, Object> compressionConfig(boolean clientCompression, boolean useHttpCompression,
+                                                         boolean sendParamsInBody) {
+        Map<String, Object> reqConfig = new HashMap<>();
+        reqConfig.put(ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey(), clientCompression);
+        reqConfig.put(ClientConfigProperties.USE_HTTP_COMPRESSION.getKey(), useHttpCompression);
+        reqConfig.put(ClientConfigProperties.HTTP_SEND_PARAMS_IN_BODY.getKey(), sendParamsInBody);
+        return reqConfig;
+    }
+
+    private static String headerValue(HttpPost req, String name) {
+        Header header = req.getFirstHeader(name);
+        return header == null ? null : header.getValue();
+    }
+
     /**
      * A server error is logged at WARN only for an unknown status code (the switch's default branch). Known
      * error paths emit no server-error WARN: readError surfaces an exception-code error, a mapped code (502)