[jdbc-v2,client-v2] Fix format selection and document how to

CHANGELOG.md

@@ -153,6 +153,7 @@
 
 ### Bug Fixes 
 
+- **[jdbc-v2, client-v2]** Fixes issue with `FORMAT` in query unable to override format set by client when used with ClickHouse 26.8+. Default format is `RowBinaryWithNamesAndTypes` set at client level. For JDBC, recommend using `format=JSONEachRow` to query JSON. Setting `format=` (empty or `null`) omits the format request header so explicit query `FORMAT` clauses take effect; note that on JDBC any statement without a `FORMAT` clause and all `DatabaseMetaData` operations will fail because the server falls back to `default_format` (`TabSeparated`). (https://github.com/ClickHouse/clickhouse-java/issues/3086)
 - **[jdbc-v2]** Fixed `Connection#prepareStatement` and `PreparedStatement#addBatch` throwing
   `StringIndexOutOfBoundsException` for an `INSERT ... VALUES (...)` statement containing a JDBC escape sequence
   (`{d '...'}`, `{ts '...'}`, ...) or a ClickHouse query parameter whose name starts with `d`/`t` (e.g. `{d:Int32}`).

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java

@@ -103,6 +103,29 @@ public enum ClickHouseFormat {
     Vertical(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#vertical
     XML(false, true, false, false, false); // https://clickhouse.com/docs/en/interfaces/formats/#xml
 
+    /**
+     * Finds ClickHouseFormat matching the given format name (case-insensitive).
+     *
+     * @param format format name, can be null or empty
+     * @return ClickHouseFormat or null if format is null or empty
+     * @throws IllegalArgumentException if format is unknown
+     */
+    public static ClickHouseFormat fromString(String format) {
+        if (format == null) {
+            return null;
+        }
+        String trimmed = format.trim();
+        if (trimmed.isEmpty()) {
+            return null;
+        }
+        for (ClickHouseFormat f : values()) {
+            if (f.name().equalsIgnoreCase(trimmed)) {
+                return f;
+            }
+        }
+        throw new IllegalArgumentException("No enum constant " + ClickHouseFormat.class.getName() + "." + trimmed);
+    }
+
     /**
      * Gets format based on given file name.
      *

clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFormatTest.java

@@ -0,0 +1,28 @@
+package com.clickhouse.data;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ClickHouseFormatTest {
+
+    @Test(groups = { "unit" })
+    public void testFromStringNullAndEmpty() {
+        Assert.assertNull(ClickHouseFormat.fromString(null));
+        Assert.assertNull(ClickHouseFormat.fromString(""));
+        Assert.assertNull(ClickHouseFormat.fromString("   "));
+    }
+
+    @Test(groups = { "unit" })
+    public void testFromStringValid() {
+        Assert.assertEquals(ClickHouseFormat.fromString("CSV"), ClickHouseFormat.CSV);
+        Assert.assertEquals(ClickHouseFormat.fromString("csv"), ClickHouseFormat.CSV);
+        Assert.assertEquals(ClickHouseFormat.fromString("  jsoneachrow  "), ClickHouseFormat.JSONEachRow);
+        Assert.assertEquals(ClickHouseFormat.fromString("RowBinaryWithNamesAndTypes"), ClickHouseFormat.RowBinaryWithNamesAndTypes);
+        Assert.assertEquals(ClickHouseFormat.fromString("rowbinarywithnamesandtypes"), ClickHouseFormat.RowBinaryWithNamesAndTypes);
+    }
+
+    @Test(groups = { "unit" })
+    public void testFromStringInvalid() {
+        Assert.expectThrows(IllegalArgumentException.class, () -> ClickHouseFormat.fromString("invalid_format_name_123"));
+    }
+}

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -59,6 +59,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -90,8 +91,6 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import javax.net.ssl.SSLContext;
-
 /**
  * <p>Client is the starting point for all interactions with ClickHouse. </p>
  *
@@ -402,7 +401,11 @@ public Builder setOption(String key, String value) {
                         + "' cannot be set as a string; supply a javax.net.ssl.SSLContext object via "
                         + "Client.Builder.setSSLContext(...)");
             }
-            this.configuration.put(key, value);
+            if (value == null) {
+                this.configuration.remove(key);
+            } else {
+                this.configuration.put(key, value);
+            }
             if (key.equals(ClientConfigProperties.PRODUCT_NAME.getKey())) {
                 setClientName(value);
             }
@@ -1301,6 +1304,30 @@ public Builder setMetricsRecorder(MetricsRecorder metricsRecorder) {
             return this;
         }
 
+        /**
+         * Sets default format used when no format is specified in {@code QuerySettings}.
+         * Accepts a ClickHouse format name as a String (e.g. "RowBinaryWithNamesAndTypes", "CSV", "JSONEachRow").
+         * String input is accepted to allow usage of new ClickHouse formats not yet defined in {@link ClickHouseFormat}.
+         * Pass {@code null} or an empty string to send no format header.
+         *
+         * @param format - ClickHouse format name, or null / empty string for no format header
+         * @return this instance of builder
+         */
+        public Builder queryFormat(String format) {
+            if (format == null || format.trim().isEmpty()) {
+                this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), null);
+                return this;
+            }
+            String trimmed = format.trim();
+            try {
+                ClickHouseFormat chFormat = ClickHouseFormat.fromString(trimmed);
+                this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), chFormat.name());
+            } catch (IllegalArgumentException e) {
+                this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), trimmed);
+            }
+            return this;
+        }
+
         public Client build() {
             // check if endpoint are empty. so can not initiate client
             if (this.endpoints.isEmpty()) {
@@ -1908,9 +1935,6 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
         }
         final QuerySettings requestSettings = new QuerySettings(buildRequestSettings(settings.getAllSettings()));
 
-        if (requestSettings.getFormat() == null) {
-            requestSettings.setFormat(ClickHouseFormat.RowBinaryWithNamesAndTypes);
-        }
         applyFormatSpecificSettings(requestSettings);
         ClientStatisticsHolder clientStats = new ClientStatisticsHolder();
         // Origin of the duration of a failed operation. Taken where the client starts OP_DURATION, which is

client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java

@@ -125,7 +125,7 @@ public enum ClientConfigProperties {
 
     RETRY_ON_FAILURE("retry", Integer.class, "3"),
 
-    INPUT_OUTPUT_FORMAT("format", ClickHouseFormat.class),
+    INPUT_OUTPUT_FORMAT("format", ClickHouseFormat.class, ClickHouseFormat.RowBinaryWithNamesAndTypes.name()),
 
     MAX_THREADS_PER_CLIENT("max_threads_per_client", Integer.class, "0"),
 
@@ -347,9 +347,20 @@ public Object parseValue(String value) {
         }
 
         if (valueType.isEnum()) {
+            String configValue = value.trim();
+            if (configValue.isEmpty()) {
+                return null;
+            }
+            if (valueType.equals(ClickHouseFormat.class)) {
+                try {
+                    return ClickHouseFormat.fromString(configValue);
+                } catch (IllegalArgumentException e) {
+                    return configValue;
+                }
+            }
             Object[] constants = valueType.getEnumConstants();
             for (Object constant : constants) {
-                if (constant.toString().equals(value)) {
+                if (constant.toString().equalsIgnoreCase(configValue)) {
                     return constant;
                 }
             }
@@ -395,7 +406,9 @@ public static Map<String, Object> parseConfigMap(Map<String, String> configMap)
                     default:
                         parsedValue = config.parseValue(value);
                 }
-                parsedConfig.put(config.getKey(), parsedValue);
+                if (parsedValue != null) {
+                    parsedConfig.put(config.getKey(), parsedValue);
+                }
             }
         }
 

client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java

@@ -872,10 +872,16 @@ private void logServerErrorResponse(HttpPost req, ClassicHttpResponse httpRespon
     private void addHeaders(HttpPost req, Map<String, Object> requestConfig) {
         setHeader(req, HttpHeaders.CONTENT_TYPE, CONTENT_TYPE.getMimeType());
         if (requestConfig.containsKey(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())) {
-            setHeader(
-                req,
-                ClickHouseHttpProto.HEADER_FORMAT,
-                    ((ClickHouseFormat) requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())).name());
+            Object formatObj = requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey());
+            if (formatObj != null) {
+                String formatStr = formatObj instanceof ClickHouseFormat ? ((ClickHouseFormat) formatObj).name() : formatObj.toString();
+                if (!formatStr.trim().isEmpty()) {
+                    setHeader(
+                        req,
+                        ClickHouseHttpProto.HEADER_FORMAT,
+                        formatStr);
+                }
+            }
         }
         if (requestConfig.containsKey(ClientConfigProperties.QUERY_ID.getKey())) {
             setHeader(

client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java

@@ -82,6 +82,12 @@ public void close() throws Exception {
         }
     }
 
+    /**
+     * Returns format of the date stream accessible via {@link #getInputStream()}
+     * This format is set from server response header `X-ClickHouse-Format`.
+     *
+     * @return ClickHouseFormat - format matching server response format.
+     */
     public ClickHouseFormat getFormat() {
         return format;
     }

client-v2/src/test/java/com/clickhouse/client/ClientTests.java

@@ -333,7 +333,7 @@ public void testDefaultSettings() {
                     Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match");
                 }
             }
-            Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added.
+            Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added.
         }
 
         try (Client client = new Client.Builder()
@@ -365,9 +365,10 @@ public void testDefaultSettings() {
                 .setSocketRcvbuf(100000)
                 .setSocketSndbuf(100000)
                 .binaryStringSupport(true)
+                .queryFormat(ClickHouseFormat.CSV.name())
                 .build()) {
             Map<String, String> config = client.getConfiguration();
-            Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added.
+            Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added.
             Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb");
             Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10");
             Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000");
@@ -393,7 +394,7 @@ public void testDefaultSettings() {
             Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_SNDBUF_OPT.getKey()), "100000");
             Assert.assertEquals(config.get(ClientConfigProperties.SSL_MODE.getKey()), "STRICT");
             Assert.assertEquals(config.get(ClientConfigProperties.BINARY_STRING_SUPPORT.getKey()), "true");
-
+            Assert.assertEquals(config.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CSV");
         }
     }
 
@@ -437,7 +438,7 @@ public void testWithOldDefaults() {
                     Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match");
                 }
             }
-            Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added.
+            Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added.
         }
     }
 
@@ -734,6 +735,61 @@ public void testInvalidAuthConfiguration() throws Exception {
                 Assert.assertTrue(e.getMessage().contains("Trust store and certificates cannot be used together"), e.getMessage()));
     }
 
+    @Test
+    public void testFormatPropertyParsing() {
+        Map<String, String> rawMap = new HashMap<>();
+        rawMap.put("format", "csv");
+        Map<String, Object> parsedMap = ClientConfigProperties.parseConfigMap(rawMap);
+        Assert.assertEquals(parsedMap.get("format"), ClickHouseFormat.CSV);
+
+        rawMap.clear();
+        rawMap.put("format", "  jsoneachrow  ");
+        parsedMap = ClientConfigProperties.parseConfigMap(rawMap);
+        Assert.assertEquals(parsedMap.get("format"), ClickHouseFormat.JSONEachRow);
+
+        rawMap.clear();
+        rawMap.put("format", "CustomNewFormat");
+        parsedMap = ClientConfigProperties.parseConfigMap(rawMap);
+        Assert.assertEquals(parsedMap.get("format"), "CustomNewFormat");
+
+        rawMap.clear();
+        rawMap.put("format", "");
+        parsedMap = ClientConfigProperties.parseConfigMap(rawMap);
+        Assert.assertFalse(parsedMap.containsKey("format"), "Empty string format should result in key not present or null");
+
+        rawMap.clear();
+        rawMap.put("format", "   ");
+        parsedMap = ClientConfigProperties.parseConfigMap(rawMap);
+        Assert.assertFalse(parsedMap.containsKey("format"), "Whitespace format should result in key not present or null");
+    }
+
+    @Test
+    public void testQueryFormatBuilder() {
+        try (Client c1 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat(null).build()) {
+            Assert.assertNull(c1.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()));
+        }
+
+        try (Client c2 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("").build()) {
+            Assert.assertNull(c2.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()));
+        }
+
+        try (Client c3 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("   ").build()) {
+            Assert.assertNull(c3.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()));
+        }
+
+        try (Client c4 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("csv").build()) {
+            Assert.assertEquals(c4.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CSV");
+        }
+
+        try (Client c5 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("  jsoneachrow  ").build()) {
+            Assert.assertEquals(c5.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "JSONEachRow");
+        }
+
+        try (Client c6 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("CustomNewFormat").build()) {
+            Assert.assertEquals(c6.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CustomNewFormat");
+        }
+    }
+
     @Test(groups = {"integration"})
     public void testOverrideSettings() throws Exception {
         final String clientTimezone = "America/Los_Angeles";

client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java

@@ -2383,15 +2383,55 @@ public void testEmptyResponse() throws Exception {
 
     @Test(groups = {"integration"})
     public void testSettingsNotChanged() throws Exception{
-        final QuerySettings settings = Mockito.spy(new QuerySettings());
-        try (QueryResponse response = client.query("select 1 FORMAT JSONEachRow", settings).get()) {
+        final QuerySettings settings = Mockito.spy(new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow));
+        try (QueryResponse response = client.query("select 1", settings).get()) {
             Mockito.verify(settings, Mockito.times(1)).getAllSettings();
             Mockito.verifyNoMoreInteractions(settings);
-            Assert.assertNull(settings.getFormat());
+            Assert.assertEquals(settings.getFormat(), ClickHouseFormat.JSONEachRow);
             Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow);
         }
     }
 
+    @Test(groups = {"integration"})
+    public void testFormatSelectionPrecedence() throws Exception {
+        // 1. Explicit QuerySettings format overrides client default
+        QuerySettings settingsFormat = new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow);
+        try (QueryResponse response = client.query("SELECT 1 AS num", settingsFormat).get()) {
+            Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow);
+        }
+
+        // 2. Default client format is RowBinaryWithNamesAndTypes
+        try (QueryResponse response = client.query("SELECT 1 AS num").get()) {
+            Assert.assertEquals(response.getFormat(), ClickHouseFormat.RowBinaryWithNamesAndTypes);
+        }
+
+        // 3. Client configured with format set to null or empty string allows query SQL FORMAT clause to take effect
+        try (Client nullFormatClient = newClient()
+                .queryFormat(null)
+                .build()) {
+            try (QueryResponse response = nullFormatClient.query("SELECT 1 AS num FORMAT JSONEachRow").get()) {
+                Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow);
+            }
+        }
+
+        try (Client emptyFormatClient = newClient()
+                .queryFormat("")
+                .build()) {
+            try (QueryResponse response = emptyFormatClient.query("SELECT 1 AS num FORMAT JSONEachRow").get()) {
+                Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow);
+            }
+        }
+
+        // 4. Client configured via queryFormat(...) with lowercase or custom format string
+        try (Client customFormatClient = newClient()
+                .queryFormat("csv")
+                .build()) {
+            try (QueryResponse response = customFormatClient.query("SELECT 1 AS num").get()) {
+                Assert.assertEquals(response.getFormat(), ClickHouseFormat.CSV);
+            }
+        }
+    }
+
     @Test
     public void testDuplicateColumnNames() throws Exception {
         {

docs/clickhouse-docs/client.mdx

@@ -585,6 +585,22 @@ This object should be closed as soon as possible to release a connection because
 
 ## Query API {#query-api}
 
+**Format Selection**
+
+Client provides transparent access to the response stream from server. User is free to choose any ClickHouse format in request and read data via `InputStream`
+from `QueryResponse` object. 
+Format can be requested via: 
+- `QuerySettings#setFormat()` this will set format header in a request. 
+- In `FORMAT` clause in query itself. 
+- Server settings `default_format` (/reference/settings/session-settings/default#default_format)
+
+There are differences in behavior depending on ClickHouse and Client version: 
+- Client < `0.11.0` & ClickHouse < `26.8` - `FORMAT` clause has priority over request format header.
+- Client > `0.11.0` & ClickHouse > `26.8` - request header has priority over `FORMAT` clause.
+    - Client sets header when no format is specified in `QuerySettings` - this doesn't allow to override format in query. 
+- Client starting `0.11.0` - sets default `format` value on client level and not on operation level. This allows existig code 
+work without changes and new code to use `FORMAT` clause by settings `format` on client to `null`.
+
 ### query(String sqlQuery) {#querystring-sqlquery}
 
 Sends `sqlQuery` as is. Response format is set by query settings. `QueryResponse` will hold a reference to the response stream that should be consumed by a reader for the supportig format.