diff --git a/CHANGELOG.md b/CHANGELOG.md index be4351068..b264b9415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,6 +153,13 @@ ### Bug Fixes +- **[jdbc-v2, client-v2]** Fixes issue with `FORMAT` in query unable to override format set by client when used with + ClickHouse 26.8+. Default format is `RowBinaryWithNamesAndTypes` set at client level. For JDBC, recommend using + `format=JSONEachRow` to query JSON. Setting `format=` (empty or `null`) omits the format request header so explicit + query `FORMAT` clauses take effect; note that on JDBC any statement without a `FORMAT` clause will fail because the + server falls back to `default_format` (`TabSeparated`). `DatabaseMetaData` is unaffected: every statement it runs + internally pins `RowBinaryWithNamesAndTypes` in its own settings, so metadata keeps working regardless of the + connection's `format` property. (https://github.com/ClickHouse/clickhouse-java/issues/3086) - **[jdbc-v2]** Fixed `Connection#prepareStatement` and `PreparedStatement#addBatch` throwing `StringIndexOutOfBoundsException` for an `INSERT ... VALUES (...)` statement containing a JDBC escape sequence (`{d '...'}`, `{ts '...'}`, ...) or a ClickHouse query parameter whose name starts with `d`/`t` (e.g. `{d:Int32}`). diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java index 9f64e7ab4..c288cca2a 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java @@ -103,6 +103,29 @@ public enum ClickHouseFormat { Vertical(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#vertical XML(false, true, false, false, false); // https://clickhouse.com/docs/en/interfaces/formats/#xml + /** + * Finds ClickHouseFormat matching the given format name (case-insensitive). + * + * @param format format name, can be null or empty + * @return ClickHouseFormat or null if format is null or empty + * @throws IllegalArgumentException if format is unknown + */ + public static ClickHouseFormat fromString(String format) { + if (format == null) { + return null; + } + String trimmed = format.trim(); + if (trimmed.isEmpty()) { + return null; + } + for (ClickHouseFormat f : values()) { + if (f.name().equalsIgnoreCase(trimmed)) { + return f; + } + } + throw new IllegalArgumentException("No enum constant " + ClickHouseFormat.class.getName() + "." + trimmed); + } + /** * Gets format based on given file name. * diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFormatTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFormatTest.java new file mode 100644 index 000000000..5efedacb6 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFormatTest.java @@ -0,0 +1,28 @@ +package com.clickhouse.data; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseFormatTest { + + @Test(groups = { "unit" }) + public void testFromStringNullAndEmpty() { + Assert.assertNull(ClickHouseFormat.fromString(null)); + Assert.assertNull(ClickHouseFormat.fromString("")); + Assert.assertNull(ClickHouseFormat.fromString(" ")); + } + + @Test(groups = { "unit" }) + public void testFromStringValid() { + Assert.assertEquals(ClickHouseFormat.fromString("CSV"), ClickHouseFormat.CSV); + Assert.assertEquals(ClickHouseFormat.fromString("csv"), ClickHouseFormat.CSV); + Assert.assertEquals(ClickHouseFormat.fromString(" jsoneachrow "), ClickHouseFormat.JSONEachRow); + Assert.assertEquals(ClickHouseFormat.fromString("RowBinaryWithNamesAndTypes"), ClickHouseFormat.RowBinaryWithNamesAndTypes); + Assert.assertEquals(ClickHouseFormat.fromString("rowbinarywithnamesandtypes"), ClickHouseFormat.RowBinaryWithNamesAndTypes); + } + + @Test(groups = { "unit" }) + public void testFromStringInvalid() { + Assert.expectThrows(IllegalArgumentException.class, () -> ClickHouseFormat.fromString("invalid_format_name_123")); + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index dbe29a268..22b949e88 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -59,6 +59,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -90,8 +91,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import javax.net.ssl.SSLContext; - /** *

Client is the starting point for all interactions with ClickHouse.

* @@ -402,7 +401,11 @@ public Builder setOption(String key, String value) { + "' cannot be set as a string; supply a javax.net.ssl.SSLContext object via " + "Client.Builder.setSSLContext(...)"); } - this.configuration.put(key, value); + if (value == null) { + this.configuration.remove(key); + } else { + this.configuration.put(key, value); + } if (key.equals(ClientConfigProperties.PRODUCT_NAME.getKey())) { setClientName(value); } @@ -1301,6 +1304,29 @@ public Builder setMetricsRecorder(MetricsRecorder metricsRecorder) { return this; } + /** + * Sets default format used when no format is specified in {@code QuerySettings}. + * Accepts a ClickHouse format name as a String (e.g. "RowBinaryWithNamesAndTypes", "CSV", "JSONEachRow"). + * String input is accepted to allow usage of new ClickHouse formats not yet defined in {@link ClickHouseFormat}. + * Pass {@code null} or an empty string to send no format header. + * + * @param format - ClickHouse format name, or null / empty string for no format header + * @return this instance of builder + */ + public Builder queryFormat(String format) { + if (ClientUtils.isBlank(format)) { + this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), null); + return this; + } + try { + ClickHouseFormat chFormat = ClickHouseFormat.fromString(format); + this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), chFormat.name()); + } catch (IllegalArgumentException e) { + this.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), format.trim()); + } + return this; + } + public Client build() { // check if endpoint are empty. so can not initiate client if (this.endpoints.isEmpty()) { @@ -1908,9 +1934,6 @@ public CompletableFuture query(String sqlQuery, Map parseConfigMap(Map configMap) default: parsedValue = config.parseValue(value); } - parsedConfig.put(config.getKey(), parsedValue); + if (parsedValue != null) { + parsedConfig.put(config.getKey(), parsedValue); + } } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 54d53eec9..036ed48d2 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -872,10 +872,16 @@ private void logServerErrorResponse(HttpPost req, ClassicHttpResponse httpRespon private void addHeaders(HttpPost req, Map requestConfig) { setHeader(req, HttpHeaders.CONTENT_TYPE, CONTENT_TYPE.getMimeType()); if (requestConfig.containsKey(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())) { - setHeader( - req, - ClickHouseHttpProto.HEADER_FORMAT, - ((ClickHouseFormat) requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())).name()); + Object formatObj = requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()); + if (formatObj != null) { + String formatStr = formatObj instanceof String ? formatObj.toString() : ((ClickHouseFormat)formatObj).name(); + if (ClientUtils.isNotBlank(formatStr)) { + setHeader( + req, + ClickHouseHttpProto.HEADER_FORMAT, + formatStr); + } + } } if (requestConfig.containsKey(ClientConfigProperties.QUERY_ID.getKey())) { setHeader( diff --git a/client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java b/client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java index 6c0de2e3e..2858f85ab 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java @@ -82,6 +82,12 @@ public void close() throws Exception { } } + /** + * Returns format of the date stream accessible via {@link #getInputStream()} + * This format is set from server response header `X-ClickHouse-Format`. + * + * @return ClickHouseFormat - format matching server response format. + */ public ClickHouseFormat getFormat() { return format; } diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index b50d3700b..aef988818 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -333,7 +333,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. } try (Client client = new Client.Builder() @@ -365,9 +365,10 @@ public void testDefaultSettings() { .setSocketRcvbuf(100000) .setSocketSndbuf(100000) .binaryStringSupport(true) + .queryFormat(ClickHouseFormat.CSV.name()) .build()) { Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added. Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); @@ -393,7 +394,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_SNDBUF_OPT.getKey()), "100000"); Assert.assertEquals(config.get(ClientConfigProperties.SSL_MODE.getKey()), "STRICT"); Assert.assertEquals(config.get(ClientConfigProperties.BINARY_STRING_SUPPORT.getKey()), "true"); - + Assert.assertEquals(config.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CSV"); } } @@ -437,7 +438,7 @@ public void testWithOldDefaults() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. } } @@ -734,6 +735,61 @@ public void testInvalidAuthConfiguration() throws Exception { Assert.assertTrue(e.getMessage().contains("Trust store and certificates cannot be used together"), e.getMessage())); } + @Test + public void testFormatPropertyParsing() { + Map rawMap = new HashMap<>(); + rawMap.put("format", "csv"); + Map parsedMap = ClientConfigProperties.parseConfigMap(rawMap); + Assert.assertEquals(parsedMap.get("format"), ClickHouseFormat.CSV); + + rawMap.clear(); + rawMap.put("format", " jsoneachrow "); + parsedMap = ClientConfigProperties.parseConfigMap(rawMap); + Assert.assertEquals(parsedMap.get("format"), ClickHouseFormat.JSONEachRow); + + rawMap.clear(); + rawMap.put("format", "CustomNewFormat"); + parsedMap = ClientConfigProperties.parseConfigMap(rawMap); + Assert.assertEquals(parsedMap.get("format"), "CustomNewFormat"); + + rawMap.clear(); + rawMap.put("format", ""); + parsedMap = ClientConfigProperties.parseConfigMap(rawMap); + Assert.assertFalse(parsedMap.containsKey("format"), "Empty string format should result in key not present or null"); + + rawMap.clear(); + rawMap.put("format", " "); + parsedMap = ClientConfigProperties.parseConfigMap(rawMap); + Assert.assertFalse(parsedMap.containsKey("format"), "Whitespace format should result in key not present or null"); + } + + @Test + public void testQueryFormatBuilder() { + try (Client c1 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat(null).build()) { + Assert.assertNull(c1.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())); + } + + try (Client c2 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("").build()) { + Assert.assertNull(c2.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())); + } + + try (Client c3 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat(" ").build()) { + Assert.assertNull(c3.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())); + } + + try (Client c4 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("csv").build()) { + Assert.assertEquals(c4.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CSV"); + } + + try (Client c5 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat(" jsoneachrow ").build()) { + Assert.assertEquals(c5.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "JSONEachRow"); + } + + try (Client c6 = new Client.Builder().addEndpoint("http://localhost:8123").queryFormat("CustomNewFormat").build()) { + Assert.assertEquals(c6.getConfiguration().get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey()), "CustomNewFormat"); + } + } + @Test(groups = {"integration"}) public void testOverrideSettings() throws Exception { final String clientTimezone = "America/Los_Angeles"; diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index 08ada16db..d26f59764 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -2383,15 +2383,55 @@ public void testEmptyResponse() throws Exception { @Test(groups = {"integration"}) public void testSettingsNotChanged() throws Exception{ - final QuerySettings settings = Mockito.spy(new QuerySettings()); - try (QueryResponse response = client.query("select 1 FORMAT JSONEachRow", settings).get()) { + final QuerySettings settings = Mockito.spy(new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow)); + try (QueryResponse response = client.query("select 1", settings).get()) { Mockito.verify(settings, Mockito.times(1)).getAllSettings(); Mockito.verifyNoMoreInteractions(settings); - Assert.assertNull(settings.getFormat()); + Assert.assertEquals(settings.getFormat(), ClickHouseFormat.JSONEachRow); Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow); } } + @Test(groups = {"integration"}) + public void testFormatSelectionPrecedence() throws Exception { + // 1. Explicit QuerySettings format overrides client default + QuerySettings settingsFormat = new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow); + try (QueryResponse response = client.query("SELECT 1 AS num", settingsFormat).get()) { + Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow); + } + + // 2. Default client format is RowBinaryWithNamesAndTypes + try (QueryResponse response = client.query("SELECT 1 AS num").get()) { + Assert.assertEquals(response.getFormat(), ClickHouseFormat.RowBinaryWithNamesAndTypes); + } + + // 3. Client configured with format set to null or empty string allows query SQL FORMAT clause to take effect + try (Client nullFormatClient = newClient() + .queryFormat(null) + .build()) { + try (QueryResponse response = nullFormatClient.query("SELECT 1 AS num FORMAT JSONEachRow").get()) { + Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow); + } + } + + try (Client emptyFormatClient = newClient() + .queryFormat("") + .build()) { + try (QueryResponse response = emptyFormatClient.query("SELECT 1 AS num FORMAT JSONEachRow").get()) { + Assert.assertEquals(response.getFormat(), ClickHouseFormat.JSONEachRow); + } + } + + // 4. Client configured via queryFormat(...) with lowercase or custom format string + try (Client customFormatClient = newClient() + .queryFormat("csv") + .build()) { + try (QueryResponse response = customFormatClient.query("SELECT 1 AS num").get()) { + Assert.assertEquals(response.getFormat(), ClickHouseFormat.CSV); + } + } + } + @Test public void testDuplicateColumnNames() throws Exception { { diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index 6a2ff98b2..4284bed3f 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -585,6 +585,22 @@ This object should be closed as soon as possible to release a connection because ## Query API {#query-api} +**Format Selection** + +Client provides transparent access to the response stream from server. User is free to choose any ClickHouse format in request and read data via `InputStream` +from `QueryResponse` object. +Format can be requested via: +- `QuerySettings#setFormat()` this will set format header in a request. +- In `FORMAT` clause in query itself. +- Server settings `default_format` (/reference/settings/session-settings/default#default_format) + +There are differences in behavior depending on ClickHouse and Client version: +- Client < `0.11.0` & ClickHouse < `26.8` - `FORMAT` clause has priority over request format header. +- Client > `0.11.0` & ClickHouse > `26.8` - request header has priority over `FORMAT` clause. + - Client sets header when no format is specified in `QuerySettings` - this doesn't allow to override format in query. +- Client starting `0.11.0` - sets default `format` value on client level and not on operation level. This allows existig code +work without changes and new code to use `FORMAT` clause by settings `format` on client to `null`. + ### query(String sqlQuery) {#querystring-sqlquery} Sends `sqlQuery` as is. Response format is set by query settings. `QueryResponse` will hold a reference to the response stream that should be consumed by a reader for the supportig format. diff --git a/docs/features.md b/docs/features.md index ec6c568f4..78df8b01d 100644 --- a/docs/features.md +++ b/docs/features.md @@ -14,7 +14,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Proxy support: Can send requests through configured HTTP proxies, including proxy credentials. - Connection and socket tuning: Exposes pool sizing, keep-alive, reuse strategy, connect/request/socket timeouts, and low-level socket options. - Query execution: Executes SQL asynchronously and returns streaming query responses with response metadata and metrics. -- Query settings: Supports per-query database selection, output format, execution limits, roles, log comments, headers, reusable `Session` objects, session settings, server settings, and network timeout overrides. Settings explicitly set to `null` will not be sent to the server. The output format must be set through the query settings (`QuerySettings#setFormat`) and not with a `FORMAT` clause in the query: the client always sends the format of the settings in the `X-ClickHouse-Format` header, and a `26.8+` server uses that header in preference to a `FORMAT` clause in the query. +- Query settings: Supports per-query database selection, output format, execution limits, roles, log comments, headers, reusable `Session` objects, session settings, server settings, and network timeout overrides. Settings explicitly set to `null` or empty string will not be sent to the server. The default format (`RowBinaryWithNamesAndTypes`) is configured at the client level (`ClientConfigProperties.INPUT_OUTPUT_FORMAT`), sending the `X-ClickHouse-Format` request header. ClickHouse `26.8+` uses that request header in preference to a `FORMAT` clause in the SQL query. Configuring `format=` (empty) or `null` omits the format request header, allowing explicit SQL `FORMAT` clauses to take effect; however, on a JDBC connection this is an expert setting, as queries without a `FORMAT` clause fall back to server `default_format` (`TabSeparated`), which JDBC rejects. - Parameterized SQL: Accepts named query parameters and can send them through supported HTTP request encodings. - Result materialization helpers: Provides streaming `Records`, generic row access, and convenience APIs that materialize all rows into generic records or typed POJOs. A `Nullable(T)` column may be bound to a POJO field of a primitive type (e.g. `long`), which reads any non-`NULL` value without boxing; a value that is actually `NULL` cannot be held by such a field and is reported with a `NullValueException` (use the boxed type to accept `NULL`). - Binary format readers: Reads ClickHouse binary result formats including `Native`, `RowBinary`, `RowBinaryWithNames`, and `RowBinaryWithNamesAndTypes`. @@ -109,7 +109,7 @@ Compatibility-sensitive traits: - Binary access to `String`/`FixedString` columns is compatibility-sensitive: `getBytes(...)` and `getBinaryStream(...)` expose the raw column bytes (not a re-encoded text literal), and a `NULL` column returns `null` with `wasNull()` reporting `true`. The `binary_string_support` connection property is passed through to the underlying `client-v2` transport. - `Geometry` has a stable JDBC mapping: metadata reports SQL type `ARRAY` with type name `Geometry`, read paths return nested Java arrays rather than custom wrappers, and write paths depend on the caller preserving the intended point/array nesting shape. - JDBC `Geometry` writes share the same ambiguity as the client serializer: variant selection is inferred from nesting depth, so `Ring` versus `LineString` versus `MultiPoint`, and `Polygon` versus `MultiLineString`, are not currently distinguishable when writing through the generic `Geometry` path. -- JDBC `FORMAT JSONEachRow` support is opt-in through the `jdbc_json_parser_factory` driver property, whose value must be a fully-qualified `JsonParserFactory` class name with a public no-argument constructor; JSONEachRow numeric and structured value behavior follows the selected parser and configured server output settings. Inferred JSON arrays are returned from `ResultSet.getObject(...)` as parser-native `List` values rather than JDBC `Array` values because JSONEachRow does not include element metadata. JDBC temporal typed accessors such as `getTimestamp(...)` are not guaranteed for JSONEachRow result sets; callers that need stable JDBC temporal conversions should use the binary default format or perform application-level conversion from string/object values. +- JDBC `FORMAT JSONEachRow` support is opt-in through the `jdbc_json_parser_factory` driver property, whose value must be a fully-qualified `JsonParserFactory` class name with a public no-argument constructor; JSONEachRow numeric and structured value behavior follows the selected parser and configured server output settings. Inferred JSON arrays are returned from `ResultSet.getObject(...)` as parser-native `List` values rather than JDBC `Array` values because JSONEachRow does not include element metadata. JDBC temporal typed accessors such as `getTimestamp(...)` are not guaranteed for JSONEachRow result sets; callers that need stable JDBC temporal conversions should use the binary default format or perform application-level conversion from string/object values. The recommended way to read JSON in JDBC is configuring `format=JSONEachRow` in connection properties (along with `jdbc_json_parser_factory`). Setting `format=` (empty or `null`) is an option: while it permits SQL query `FORMAT` clauses to take effect, any statement without an explicit `FORMAT` clause will fail because ClickHouse server falls back to `default_format` (`TabSeparated` or what set in `clickhouse_setting_default_format` on the connection). `DatabaseMetaData` methods are exempt — they pin `RowBinaryWithNamesAndTypes` in the settings of the statements they run internally, so metadata works for any value of `format`. - Standard `FORMAT JSON` output has ClickHouse-specific `meta` and `data` fields and is not exposed as a JDBC `ResultSet`. JDBC callers that need it should unwrap to `ConnectionImpl`, call `getClient()`, and parse the `QueryResponse` stream directly. - Binary parameters passed through `setBytes()` are encoded as ClickHouse `unhex(...)` expressions rather than text literals; empty byte arrays map to an empty string expression. - Stream and reader setters (`setAsciiStream`, `setUnicodeStream`, `setBinaryStream`, `setCharacterStream`, `setNCharacterStream`) are treated as text input encoded with the same string-escaping rules, including length-based truncation when a length is supplied. diff --git a/docs/integration-client.md b/docs/integration-client.md index 05df55e08..4fcb08127 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -492,6 +492,40 @@ Consider these trade-offs: Always pick the format that minimizes unnecessary transcoding in your application layer. +### Format Selection + +The client provides transparent access to the response stream from ClickHouse. You can request any supported ClickHouse format in your request and read data via the `InputStream` from the `QueryResponse` object. + +A response format can be specified in several ways: +- **`QuerySettings#setFormat(ClickHouseFormat format)`**: Sets the format header (`X-ClickHouse-Format`) for a specific query request. +- **`FORMAT` clause in SQL**: Appending `FORMAT ` directly in the SQL query string. +- **Client default setting**: The client sets a default `format` option (`ClientConfigProperties.INPUT_OUTPUT_FORMAT`, defaulting to `RowBinaryWithNamesAndTypes`) at the client level. Configured via `Client.Builder#queryFormat(String format)` or `setOption("format", ...)`. Note that `queryFormat` accepts a `String` to allow using new ClickHouse formats not yet defined in the `ClickHouseFormat` enum, and passing `null` or an empty string omits the format header. +- **Server setting**: ClickHouse server session setting (`default_format`). + +**Precedence and Version Differences:** + +- **Client < 0.11.0 & ClickHouse < 26.8:** The `FORMAT` clause in the query string takes priority over the request format header. +- **Client >= 0.11.0 & ClickHouse >= 26.8:** The request format header (`X-ClickHouse-Format`) takes priority over the `FORMAT` clause in the query string. +- **Client 0.11.0+:** Default `format` is set at the client level rather than at the operation level. This allows existing code to work without changes, while new code can use a SQL `FORMAT` clause by setting `format` on the client or in `QuerySettings` to `null`. + +**Inspecting Server Response Format:** + +Use `QueryResponse#getFormat()` to inspect the format of the response data stream (resolved from the server `X-ClickHouse-Format` response header): + +```java +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.data.ClickHouseFormat; + +public ClickHouseFormat inspectQueryFormat(Client client, String sql) throws Exception { + QuerySettings settings = new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow); + try (QueryResponse response = client.query(sql, settings).get()) { + return response.getFormat(); + } +} +``` + ## Step 6 — Read operations & tuning **Goal:** read results efficiently and configure the operation-level settings for heavy analytical reads. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index 4d35fa885..11ce8f0f9 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -402,18 +402,71 @@ public boolean checkConnectionHealth(Connection conn, int timeoutSeconds) throws ## Step 4 — Formats under the hood -**Goal:** understand that JDBC hides format selection, so you can decide up front whether JDBC's fixed contract is sufficient. +**Goal:** understand how JDBC handles format selection internally and how to configure custom formats like `JSONEachRow`. -JDBC does **not** expose format selection. The driver picks formats internally by operation type: +JDBC uses the client's format selection mechanism under the hood. By default, the driver sends `X-ClickHouse-Format: RowBinaryWithNamesAndTypes` for query execution: | Operation | Internal format | Notes | |-----------|-----------------|-------| -| Query (`executeQuery`) | Binary row format from server | Converted to JDBC `ResultSet` rows | +| Query (`executeQuery`) | `RowBinaryWithNamesAndTypes` | Converted to JDBC `ResultSet` rows | | Simple INSERT via `Statement` | SQL text | `INSERT INTO t VALUES (...)` | | `PreparedStatement` INSERT | SQL text or RowBinary | RowBinary when `beta.row_binary_for_simple_insert=true` | | Writer statement INSERT | RowBinary | Streaming binary writer | | Batch INSERT | Multi-row SQL rewrite or RowBinary | Depends on statement shape | +### Format Selection and SQL `FORMAT` Clauses + +The response format can be configured using the `format` connection property (`ClientConfigProperties.INPUT_OUTPUT_FORMAT` or `"format"`). + +**Important for ClickHouse 26.8+:** +- On ClickHouse 26.8+, the request format header sent by the driver (`X-ClickHouse-Format`) takes priority over a `FORMAT` clause written in the SQL query string. +- By default, the driver sends `format=RowBinaryWithNamesAndTypes`. +- To read JSON in JDBC, the recommended approach is setting `format=JSONEachRow` in connection properties along with `jdbc_json_parser_factory`. +- Setting `format=` (empty string) or `null` is an **expert-only setting**: + - Setting `format=` omits the `X-ClickHouse-Format` request header, allowing explicit SQL `FORMAT` clauses written in query strings to take effect. + - **Caveat:** For any statement without an explicit SQL `FORMAT` clause, the server falls back to its `default_format` (`TabSeparated`). Because JDBC `ResultSet` only consumes `RowBinaryWithNamesAndTypes` and `JSONEachRow`, such queries fail with a `SQLException`. + - `DatabaseMetaData` operations (e.g. `getTables()`, `getColumns()`) are not affected by the `format` property: they pin `RowBinaryWithNamesAndTypes` on the statements they run internally. + +### Usage of `JSONEachRow` in JDBC + +JDBC V2 supports streaming `JSONEachRow` responses as standard `ResultSet` instances. This feature is opt-in and requires configuring a `JsonParserFactory`. + +1. **Configure Driver Properties:** + Set `jdbc_json_parser_factory` (`DriverProperties.JSON_PARSER_FACTORY`) to the fully-qualified class name of a `JsonParserFactory` implementation (such as `JacksonJsonParserFactory` or `GsonJsonParserFactory`). + Set `format` (`ClientConfigProperties.INPUT_OUTPUT_FORMAT`) to `"JSONEachRow"`. + +```java +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; +import com.clickhouse.jdbc.DriverProperties; + +public Properties createJsonEachRowProperties() { + Properties props = new Properties(); + props.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), JacksonJsonParserFactory.class.getName()); + props.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); + return props; +} +``` + +2. **Execute Query and Process ResultSet:** + +```java +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +public void readJsonEachRowResultSet(Connection conn) throws Exception { + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT id, name, payload FROM events ORDER BY id")) { + while (rs.next()) { + int id = rs.getInt("id"); + String name = rs.getString("name"); + Object payload = rs.getObject("payload"); // returns parser-native List/Map + } + } +} +``` + ### When JDBC's format contract is not enough | Goal | JDBC approach | Better alternative | @@ -421,7 +474,7 @@ JDBC does **not** expose format selection. The driver picks formats internally b | Simple CRUD / reporting | Standard JDBC — sufficient | — | | Bulk ingest (millions of rows) | Batch `PreparedStatement` + RowBinary beta | Java Client stream insert | | Complex type handling | `getObject()` with type map | Java Client POJO/binary readers | -| Export to a file format | Not supported via JDBC | Java Client with format selection | +| Export to a file format | Not supported via ResultSet (ResultSet requires `RowBinaryWithNamesAndTypes` or `JSONEachRow`; text formats like CSV fail) | Java Client with format selection (`conn.unwrap(ConnectionImpl.class).getClient()`) | | BI tool integration | JDBC is the right choice | — | ### Hybrid usage: dropping down to the Java Client @@ -452,7 +505,7 @@ This hybrid approach allows you to use standard JDBC for simple CRUD and metadat ### Common Pitfalls -- **No format selection API** — you cannot request `Native`, `Parquet`, or `JSONEachRow` through standard JDBC. +- **Format selection scope** — format selection can be configured via connection properties (`format=JSONEachRow` or setting `jdbc_json_parser_factory`), but standard JDBC `ResultSet` requires compatible row formats (`RowBinaryWithNamesAndTypes` or `JSONEachRow`). Other wire formats like `Native` or `Parquet` require dropping down to the Java Client. - **Row-oriented output only** — no column-oriented or parallel block consumption. - **Type mapping layer** may lose precision or structure for complex types. - **Text INSERT overhead** — default SQL-based inserts are slower than binary streaming. Use the [Java Client](integration-client.md) for maximum throughput. diff --git a/docs/releases/0_11_0.md b/docs/releases/0_11_0.md index 6c40120f9..23d7ff473 100644 --- a/docs/releases/0_11_0.md +++ b/docs/releases/0_11_0.md @@ -11,3 +11,22 @@ was removed, because the kind of the operation is always known where the metrics Metrics are created by the client, and the constructor takes an internal type (`com.clickhouse.client.api.internal.ClientStatisticsHolder`), so application code is not expected to call it. Code that does call it must pass `OperationType.QUERY` or `OperationType.INSERT`. + +## CLIENT-V2, JDBC-V2: Format Selection + +There are, generally, two ways to set format: in a SQL `FORMAT` clause or in a request header (`X-ClickHouse-Format`). +Before ClickHouse `26.8` the SQL `FORMAT` clause took precedence, but in ClickHouse `26.8+` the request header takes +priority. See issue https://github.com/ClickHouse/clickhouse-java/issues/3086 + +The client now configures its default format (`RowBinaryWithNamesAndTypes`) at the client level +(`ClientConfigProperties.INPUT_OUTPUT_FORMAT`) rather than per-operation, sending the `X-ClickHouse-Format` header on +requests. + +- **For JDBC JSON queries:** Recommend configuring `format=JSONEachRow` in connection properties (along with + `jdbc_json_parser_factory`). +- **`format=` (empty) setting:** Setting `format=` (empty string) or `null` on a connection or client omits the format + request header so ClickHouse honors an explicit `FORMAT` clause written in the SQL query string. However, for a JDBC + connection, any query without an explicit `FORMAT` clause may fail because the server falls back to `default_format` + setting (`TabSeparated`), which JDBC `ResultSet` rejects. Thus setting `clickhouse_setting_default_format=RowBinaryWithNamesAndTypes` + fixes this (JDBC do not set it by default to support read-only profiles). + diff --git a/examples/jdbc-v2-json-processors/README.md b/examples/jdbc-v2-json-processors/README.md index 3eaeb1cb7..1959cfd6d 100644 --- a/examples/jdbc-v2-json-processors/README.md +++ b/examples/jdbc-v2-json-processors/README.md @@ -81,13 +81,13 @@ Each read call in `run()` follows the same three-step shape: 2. **Customize if needed** — only inside the subclass, by overriding the protected hook. 3. **Execute** — `readAll(label, factoryClass)` opens a fresh connection - with `JSON_PARSER_FACTORY=`, runs the `SELECT ... FORMAT JSONEachRow` - and iterates the `ResultSet`. + with `JSON_PARSER_FACTORY=` and `format=JSONEachRow`, runs the + `SELECT ... FORMAT JSONEachRow` and iterates the `ResultSet`. -Because JDBC selects `JSONEachRow` through SQL text, set the JSON output -server settings explicitly on the connection when numeric accessors are used: +Configure the format and JSON output server settings explicitly on the connection: ```java +props.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_floats"), "0"); props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_decimals"), "0"); diff --git a/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java b/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java index 4b54e5c88..25e221f6f 100644 --- a/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java +++ b/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java @@ -137,6 +137,7 @@ private Properties baseProperties() { var properties = new Properties(); properties.setProperty("user", user); properties.setProperty("password", password); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); properties.setProperty(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); properties.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); properties.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_floats"), "0"); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index 9dd164a13..4583b6ef8 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -309,7 +309,8 @@ protected ResultSetImpl executeQueryImpl(String sql, QuerySettings settings) thr } ClickHouseFormatReader reader; - if (response.getFormat() == ClickHouseFormat.JSONEachRow) { + ClickHouseFormat format = response.getFormat(); + if (format == ClickHouseFormat.JSONEachRow) { if (connection.getJsonParserFactory() == null) { throw new SQLException("Response is in JSONEachRow format, but " + DriverProperties.JSON_PARSER_FACTORY.getKey() + " is not configured. Set " + @@ -317,10 +318,12 @@ protected ResultSetImpl executeQueryImpl(String sql, QuerySettings settings) thr ExceptionUtils.SQL_STATE_CLIENT_ERROR); } reader = new JSONEachRowFormatReader(connection.getJsonParserFactory().createJsonParser(response.getInputStream())); - } else if (!response.getFormat().isText()) { + } else if (format != null && !format.isText()) { reader = connection.getClient().newBinaryFormatReader(response); } else { - throw new SQLException("Only RowBinaryWithNameAndTypes and JSONEachRow are supported for output format. Please check your query.", + String formatStr = format != null ? format.name() : "unknown"; + throw new SQLException("Only RowBinaryWithNameAndTypes and JSONEachRow are supported for output format, but received format '" + + formatStr + "'. Please check your query or 'format' property configuration.", ExceptionUtils.SQL_STATE_CLIENT_ERROR); } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java index 44dd9ed4f..84bb1886c 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java @@ -3,11 +3,13 @@ import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.jdbc.ClientInfoProperties; import com.clickhouse.jdbc.ConnectionImpl; import com.clickhouse.jdbc.Driver; import com.clickhouse.jdbc.DriverProperties; import com.clickhouse.jdbc.JdbcV2Wrapper; +import com.clickhouse.jdbc.StatementImpl; import com.clickhouse.jdbc.internal.DetachedResultSet; import com.clickhouse.jdbc.internal.ExceptionUtils; import com.clickhouse.jdbc.internal.JdbcUtils; @@ -93,6 +95,22 @@ public DatabaseMetaDataImpl(ConnectionImpl connection, boolean useCatalogs, Stri this.jdbcUrl = url; } + private Statement createStatement() throws SQLException { + Statement stmt = connection.createStatement(); + if (stmt instanceof StatementImpl) { + ((StatementImpl) stmt).getLocalSettings().setFormat(ClickHouseFormat.RowBinaryWithNamesAndTypes); + } + return stmt; + } + + private PreparedStatement prepareStatement(String sql) throws SQLException { + PreparedStatement stmt = connection.prepareStatement(sql); + if (stmt instanceof StatementImpl) { + ((StatementImpl) stmt).getLocalSettings().setFormat(ClickHouseFormat.RowBinaryWithNamesAndTypes); + } + return stmt; + } + @Override public boolean allProceduresAreCallable() throws SQLException { return true; @@ -735,7 +753,7 @@ public ResultSet getProcedures(String catalog, String schemaPattern, String proc "'' AS SPECIFIC_NAME " + "LIMIT 0"; try { - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -766,7 +784,7 @@ public ResultSet getProcedureColumns(String catalog, String schemaPattern, Strin "'' AS SPECIFIC_NAME " + "LIMIT 0"; try { - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1008,7 +1026,7 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam " AND t.name LIKE ?" + engineFilter; - try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (PreparedStatement stmt = prepareStatement(sql)) { stmt.setString(1, (schemaPattern == null ? "%" : schemaPattern)); stmt.setString(2, (tableNamePattern == null ? "%" : tableNamePattern)); try (ResultSet rs = stmt.executeQuery()) { @@ -1031,7 +1049,7 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam public ResultSet getSchemas() throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name try { - return connection.createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases ORDER BY name"); + return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases ORDER BY name"); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1046,7 +1064,7 @@ public ResultSet getSchemas() throws SQLException { @Override public ResultSet getCatalogs() throws SQLException { try { - return connection.createStatement().executeQuery("SELECT 'local' AS TABLE_CAT " + (useCatalogs ? "" : " WHERE 1 = 0")); + return createStatement().executeQuery("SELECT 'local' AS TABLE_CAT " + (useCatalogs ? "" : " WHERE 1 = 0")); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1061,7 +1079,7 @@ public ResultSet getCatalogs() throws SQLException { */ @Override public ResultSet getTableTypes() throws SQLException { - try (PreparedStatement stmt = connection.prepareStatement("SELECT arrayJoin(?) AS TABLE_TYPE ORDER BY TABLE_TYPE")) { + try (PreparedStatement stmt = prepareStatement("SELECT arrayJoin(?) AS TABLE_TYPE ORDER BY TABLE_TYPE")) { stmt.setObject(1, TABLE_TYPES_SQL_ARRAY); try (ResultSet rs = stmt.executeQuery()) { return DetachedResultSet.createFromResultSet(rs, connection.getDefaultCalendar(), Collections.emptyList()); @@ -1105,7 +1123,7 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa " AND table LIKE " + SQLUtils.enquoteLiteral(tableNamePattern == null ? "%" : tableNamePattern) + " AND name LIKE " + SQLUtils.enquoteLiteral(columnNamePattern == null ? "%" : columnNamePattern) + " ORDER BY TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION"; - try (Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery(sql)) { + try (Statement statement = createStatement(); ResultSet rs = statement.executeQuery(sql)) { return DetachedResultSet.createFromResultSet(rs, connection.getDefaultCalendar(), GET_COLUMNS_RS_MUTATORS); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); @@ -1149,7 +1167,7 @@ private static String generateSqlTypeSizes(String columnName) { public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(String)) AS TABLE_CAT, " + "CAST(NULL as Nullable(String)) AS TABLE_SCHEM, " + "CAST(NULL as Nullable(String)) AS TABLE_NAME, " + @@ -1168,7 +1186,7 @@ public ResultSet getColumnPrivileges(String catalog, String schema, String table public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(String)) AS TABLE_CAT, " + "CAST(NULL as Nullable(String)) AS TABLE_SCHEM, " + "CAST(NULL as Nullable(String)) AS TABLE_NAME, " + @@ -1186,7 +1204,7 @@ public ResultSet getTablePrivileges(String catalog, String schemaPattern, String public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(Int16)) AS SCOPE, " + "CAST(NULL as Nullable(String)) AS COLUMN_NAME, " + "CAST(NULL as Nullable(Int32)) AS DATA_TYPE, " + @@ -1205,7 +1223,7 @@ public ResultSet getBestRowIdentifier(String catalog, String schema, String tabl public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(Int16)) AS SCOPE, " + "CAST(NULL as Nullable(String)) AS COLUMN_NAME, " + "CAST(NULL as Nullable(Int32)) AS DATA_TYPE, " + @@ -1235,7 +1253,7 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr "AND system.tables.database ILIKE '" + (schema == null ? "%" : schema) + "' " + "AND system.tables.name ILIKE '" + (table == null ? "%" : table) + "' " + "ORDER BY COLUMN_NAME"; - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1260,7 +1278,7 @@ public ResultSet getImportedKeys(String catalog, String schema, String table) th "CAST(NULL as Nullable(String)) AS PK_NAME, " + "CAST(NULL as Nullable(Int16)) AS DEFERRABILITY" + " LIMIT 0"; - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1270,7 +1288,7 @@ public ResultSet getImportedKeys(String catalog, String schema, String table) th public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { // ClickHouse has no notion of foreign key. This method should return empty resultset try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(String)) AS PKTABLE_CAT, " + "CAST(NULL as Nullable(String)) AS PKTABLE_SCHEM, " + "CAST(NULL as Nullable(String)) AS PKTABLE_NAME, " + @@ -1310,7 +1328,7 @@ public ResultSet getCrossReference(String parentCatalog, String parentSchema, St "CAST(NULL as Nullable(String)) AS PK_NAME, " + "CAST(NULL as Nullable(Int16)) AS DEFERRABILITY" + " LIMIT 0"; - return connection.createStatement().executeQuery("SELECT " + columns); + return createStatement().executeQuery("SELECT " + columns); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1319,7 +1337,7 @@ public ResultSet getCrossReference(String parentCatalog, String parentSchema, St @Override @SuppressWarnings({"squid:S2095"}) public ResultSet getTypeInfo() throws SQLException { - try (Statement stmt = connection.createStatement(); ResultSet rs = stmt.executeQuery(DATA_TYPE_INFO_SQL)) { + try (Statement stmt = createStatement(); ResultSet rs = stmt.executeQuery(DATA_TYPE_INFO_SQL)) { return DetachedResultSet.createFromResultSet(rs, connection.getDefaultCalendar(), GET_TYPE_INFO_MUTATORS); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); @@ -1539,7 +1557,7 @@ public ResultSet getIndexInfo(String catalog, String schema, String table, boole "CAST(NULL as Nullable(Int64)) AS PAGES, " + "CAST(NULL as Nullable(String)) AS FILTER_CONDITION " + " LIMIT 0"; - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1609,7 +1627,7 @@ public boolean supportsBatchUpdates() throws SQLException { public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery("SELECT " + + return createStatement().executeQuery("SELECT " + "CAST(NULL as Nullable(String)) AS TYPE_CAT, " + "CAST(NULL as Nullable(String)) AS TYPE_SCHEM, " + "CAST(NULL as Nullable(String)) AS TYPE_NAME, " + @@ -1652,7 +1670,7 @@ public boolean supportsGetGeneratedKeys() throws SQLException { public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT CAST(NULL as Nullable(String)) AS TYPE_CAT, " + "CAST(NULL as Nullable(String)) AS TYPE_SCHEM, " + "CAST(NULL as Nullable(String)) AS TYPE_NAME, " @@ -1669,7 +1687,7 @@ public ResultSet getSuperTypes(String catalog, String schemaPattern, String type public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT " + "CAST(NULL as Nullable(String)) AS TABLE_CAT, " + "CAST(NULL as Nullable(String)) AS TABLE_SCHEM, " @@ -1685,7 +1703,7 @@ public ResultSet getSuperTables(String catalog, String schemaPattern, String tab public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, String attributeNamePattern) throws SQLException { //Return an empty result set with the required columns try { - return connection.createStatement().executeQuery( + return createStatement().executeQuery( "SELECT " + "CAST(NULL as Nullable(String)) AS TYPE_CAT, " + "CAST(NULL as Nullable(String)) AS TYPE_SCHEM, " @@ -1779,7 +1797,7 @@ public RowIdLifetime getRowIdLifetime() throws SQLException { public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { // TODO: handle useCatalogs == true and return schema catalog name try { - return connection.createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases " + + return createStatement().executeQuery("SELECT name AS TABLE_SCHEM, " + catalogPlaceholder + " AS TABLE_CATALOG FROM system.databases " + "WHERE name LIKE '" + (schemaPattern == null ? "%" : schemaPattern) + "'"); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); @@ -1815,7 +1833,7 @@ private static String getClientInfoPropertiesSql() { @Override public ResultSet getClientInfoProperties() throws SQLException { try { - return connection.createStatement().executeQuery(CLIENT_INFO_PROPERTIES_SQL); + return createStatement().executeQuery(CLIENT_INFO_PROPERTIES_SQL); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1833,7 +1851,7 @@ public ResultSet getFunctions(String catalog, String schemaPattern, String funct "FROM system.functions " + "WHERE name LIKE '" + (functionNamePattern == null ? "%" : functionNamePattern) + "'"; try { - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1862,7 +1880,7 @@ public ResultSet getFunctionColumns(String catalog, String schemaPattern, String "LIMIT 0"; try { - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -1886,7 +1904,7 @@ public ResultSet getPseudoColumns(String catalog, String schemaPattern, String t " LIMIT 0"; try { - return connection.createStatement().executeQuery(sql); + return createStatement().executeQuery(sql); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index d52adc967..677f51add 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -935,7 +935,9 @@ public void testUnwrapping() throws Exception { @Test(groups = { "integration" }) public void testRawJSONQueryThroughUnderlyingClient() throws Exception { ObjectMapper mapper = new ObjectMapper(); - try (Connection conn = getJdbcConnection(); + Properties config = new Properties(); + config.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); + try (Connection conn = getJdbcConnection(config); QueryResponse response = conn.unwrap(ConnectionImpl.class).getClient() .query("SELECT 1 AS x FORMAT JSON") .get()) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java index 54f4d0624..1f1629faf 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java @@ -297,8 +297,9 @@ public void testJsonEachRowCursorPositionDetectsLastRow() throws SQLException { public void testJsonEachRowGetObjectReturnsParserNativeArray() throws SQLException { Properties properties = new Properties(); properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), JacksonJsonParserFactory.class.getName()); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); try (Connection conn = getJdbcConnection(properties); Statement stmt = conn.createStatement()) { - try (ResultSet rs = stmt.executeQuery("SELECT [1, 2, 3] AS arr FORMAT JSONEachRow")) { + try (ResultSet rs = stmt.executeQuery("SELECT [1, 2, 3] AS arr")) { Assert.assertTrue(rs.next()); Object value = rs.getObject("arr"); Assert.assertTrue(value instanceof List, "Expected parser-native List but got " + value.getClass()); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index e4a4e5c08..7d9d7f461 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -7,6 +7,7 @@ import com.clickhouse.client.api.data_formats.JsonParserFactory; import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.query.GenericRecord; +import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.data.ClickHouseVersion; import com.clickhouse.jdbc.internal.SqlParserFacade; import org.apache.commons.lang3.RandomStringUtils; @@ -862,10 +863,57 @@ public void testCancelInsertWithSession() throws Exception { } @Test(groups = {"integration"}) - public void testTextFormatInResponse() throws Exception { - try (Connection conn = getJdbcConnection(); + public void testUnsupportedFormat() throws Exception { + Properties config1 = new Properties(); + config1.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); + try (Connection conn = getJdbcConnection(config1); Statement stmt = conn.createStatement()) { - Assert.expectThrows(SQLException.class, () -> stmt.executeQuery("SELECT 1 FORMAT JSON")); + SQLException ex = Assert.expectThrows(SQLException.class, () -> stmt.executeQuery("SELECT 1 FORMAT JSON")); + assertTrue(ex.getMessage().contains("received format 'JSON'"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("'format' property configuration"), "Unexpected message: " + ex.getMessage()); + } + + Properties config2 = new Properties(); + config2.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ClickHouseFormat.CSV.name()); + try (Connection conn = getJdbcConnection(config2); + Statement stmt = conn.createStatement()) { + SQLException ex = Assert.expectThrows(SQLException.class, () -> stmt.executeQuery("SELECT 1")); + assertTrue(ex.getMessage().contains("received format 'CSV'"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("'format' property configuration"), "Unexpected message: " + ex.getMessage()); + } + } + + @Test(groups = {"integration"}) + public void testEmptyFormatConfigurationBehavior() throws Exception { + Properties config = new Properties(); + config.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); + try (Connection conn = getJdbcConnection(config); + Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS test_empty_format_tb (id Int32, name String) ENGINE = Memory"); + try { + int updateCount = stmt.executeUpdate("INSERT INTO test_empty_format_tb VALUES (1, 'hello')"); + assertEquals(updateCount, 1); + + SQLException exQuery = Assert.expectThrows(SQLException.class, () -> stmt.executeQuery("SELECT 1")); + assertTrue(exQuery.getMessage().contains("received format 'TabSeparated'"), "Unexpected message: " + exQuery.getMessage()); + + SQLException exExec = Assert.expectThrows(SQLException.class, () -> stmt.execute("SELECT 1")); + assertTrue(exExec.getMessage().contains("received format 'TabSeparated'"), "Unexpected message: " + exExec.getMessage()); + + try (ResultSet rsMeta = conn.getMetaData().getTables(null, null, "test_empty_format_tb", null)) { + assertTrue(rsMeta.next()); + assertEquals(rsMeta.getString("TABLE_NAME"), "test_empty_format_tb"); + } + + try (ResultSet rs = stmt.executeQuery("SELECT 1 AS num FORMAT RowBinaryWithNamesAndTypes")) { + assertTrue(rs.next()); + assertEquals(rs.getInt("num"), 1); + assertFalse(rs.next()); + } + } finally { + stmt.execute("DROP TABLE IF EXISTS test_empty_format_tb"); + } } } @@ -873,6 +921,24 @@ public void testTextFormatInResponse() throws Exception { public void testJSONEachRowFormat(Class parserFactory) throws Exception { Properties properties = new Properties(); properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), parserFactory.getName()); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); + try (Connection conn = getJdbcConnection(properties)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT 1 AS num, 'test' AS str")) { + assertTrue(rs.next()); + assertEquals(rs.getInt("num"), 1); + assertEquals(rs.getString("str"), "test"); + assertFalse(rs.next()); + } + } + } + } + + @Test(groups = {"integration"}, dataProvider = "testJSONEachRowFormatDP") + public void testJSONEachRowFormatWithSqlClause(Class parserFactory) throws Exception { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), parserFactory.getName()); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); try (Connection conn = getJdbcConnection(properties)) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT 1 AS num, 'test' AS str FORMAT JSONEachRow")) { @@ -887,10 +953,12 @@ public void testJSONEachRowFormat(Class parserFactory) throws @Test(groups = {"integration"}) public void testJSONEachRowFormatRequiresParserFactory() throws Exception { - try (Connection conn = getJdbcConnection(); + Properties properties = new Properties(); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); + try (Connection conn = getJdbcConnection(properties); Statement stmt = conn.createStatement()) { try { - stmt.executeQuery("SELECT 1 AS num FORMAT JSONEachRow"); + stmt.executeQuery("SELECT 1 AS num"); fail("Expected SQLException"); } catch (SQLException e) { assertTrue(e.getMessage().contains(DriverProperties.JSON_PARSER_FACTORY.getKey()), diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index 45e7aded5..08dd650ae 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -46,6 +46,7 @@ public void testGetColumns() throws Exception { try (Connection conn = getJdbcConnection()) { final String tableName = "get_columns_metadata_test"; try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); stmt.executeUpdate("" + "CREATE TABLE " + tableName + " (id Int32, name String NOT NULL, v1 Nullable(Int8), v2 Array(Int8)) " + "ENGINE MergeTree ORDER BY tuple()"); @@ -165,6 +166,7 @@ public void testGetColumnsWithBinaryStringSupport() throws Exception { try (Connection conn = getJdbcConnection(props)) { final String tableName = "get_columns_binary_string_support_test"; try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE IF EXISTS " + tableName); stmt.executeUpdate("CREATE TABLE " + tableName + " (id Int32, name String NOT NULL, v1 Nullable(Int8), v2 Array(Int8)) " + "ENGINE MergeTree ORDER BY tuple()"); @@ -587,7 +589,7 @@ public void testGetTablesDebugSubstitutesPlaceholders() throws Exception { @Test(groups = { "integration" }) public void testGetPrimaryKeys() throws Exception { - runQuery("SELECT 1;"); + runQuery("SELECT 1 FORMAT RowBinaryWithNamesAndTypes;"); runQuery("SYSTEM FLUSH LOGS"); try (Connection conn = getJdbcConnection()) { @@ -673,6 +675,7 @@ public void testGetTablesReturnKnownTableTypes() throws Exception { } try (Statement stmt = conn.createStatement()){ + stmt.executeUpdate("DROP TABLE IF EXISTS test_db_metadata_type_memory"); stmt.executeUpdate("CREATE TABLE test_db_metadata_type_memory (v Int32) ENGINE Memory"); } try (ResultSet rs = dbmd.getTables(null, "default", "test_db_metadata_type_memory", null)) { @@ -1657,6 +1660,11 @@ public void testTableTypes() throws Exception { final DatabaseMetaData dbmd = conn.getMetaData(); try (Statement stmt = conn.createStatement()) { + // Drop views and dictionary first before dropping source table + stmt.executeUpdate("DROP DICTIONARY IF EXISTS test_table_types_dict"); + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_mat_view"); + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_view"); + // Regular MergeTree table stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_regular"); stmt.executeUpdate("CREATE TABLE test_table_types_regular (id Int32) ENGINE = MergeTree ORDER BY id"); @@ -1747,6 +1755,21 @@ public void testTableTypes() throws Exception { } } } + } finally { + try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP DICTIONARY IF EXISTS test_table_types_dict"); + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_mat_view"); + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_view"); + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_source"); + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_regular"); + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_remote"); + if (!isCloud()) { + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_log"); + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_memory"); + } + } catch (Exception e) { + // ignore cleanup errors + } } } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataWithEmptyFormatTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataWithEmptyFormatTest.java new file mode 100644 index 000000000..11cfe33c8 --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataWithEmptyFormatTest.java @@ -0,0 +1,32 @@ +package com.clickhouse.jdbc.metadata; + +import com.clickhouse.client.api.ClientConfigProperties; +import org.testng.SkipException; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +/** + * Runs the whole {@link DatabaseMetaDataTest} suite over connections configured with an empty + * {@code format} property. Such a connection omits the {@code X-ClickHouse-Format} request header, + * so the server answers with its {@code default_format} ({@code TabSeparated}) unless a statement + * asks for something else. {@code DatabaseMetaData} must keep working because it pins + * {@code RowBinaryWithNamesAndTypes} on every statement it creates. + */ +@Test(groups = { "integration" }) +public class DatabaseMetaDataWithEmptyFormatTest extends DatabaseMetaDataTest { + + @Override + public Connection getJdbcConnection(Properties properties) throws SQLException { + Properties props = properties == null ? new Properties() : (Properties) properties.clone(); + props.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), ""); + return super.getJdbcConnection(props); + } + + @Override + public void testAllTableEnginesFromSystemTableEnginesAreMapped() { + throw new SkipException("Reads through a plain Statement, which cannot consume TabSeparated"); + } +}