diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 48a0e5b0ef3..beab2f488a8 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -27,6 +27,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedActionException; @@ -319,7 +320,7 @@ protected GssApiAuthenticator( SUPPORTED_MECHANISMS, options.getAuthorizationId(), protocol, - ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(), + serverName(endPoint), options.getSaslProperties(), null); } catch (LoginException | SaslException e) { @@ -328,6 +329,23 @@ protected GssApiAuthenticator( this.endPoint = endPoint; } + /** + * The host name to build the Kerberos service principal from. + * + *

Prefers the canonical name of the resolved address, which is what Kerberos expects. The + * driver's own endpoints always hand this a resolved address — the channel carries an endpoint + * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link + * EndPoint} implementation may still yield an unresolved one, in which case {@code + * getAddress()} is null. Fall back to the host string rather than throwing a {@link + * NullPointerException}: the hostname is usually the right service name anyway, and a failed + * reverse lookup should not take authentication down. + */ + private static String serverName(EndPoint endPoint) { + InetSocketAddress address = (InetSocketAddress) endPoint.resolve(); + InetAddress inetAddress = address.getAddress(); + return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString(); + } + @NonNull @Override protected ByteBuffer getMechanism() { diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java index 168477894ed..e41079e2444 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java @@ -294,7 +294,32 @@ private Map getConnectedNodes() { .collect( Collectors.toMap( entry -> AddressFormatter.nullSafeToString(entry.getKey().getEndPoint().resolve()), - this::constructSessionStateForNode)); + this::constructSessionStateForNode, + InsightsClient::mergeNodeStates)); + } + + /** + * Combines the states of two nodes that report under the same address. + * + *

The key is not unique per node: behind an SNI proxy or a cloud client route, every node's + * endpoint resolves to the same proxy address, so any session with more than one node open + * produces duplicate keys. Without a merge function {@link Collectors#toMap} throws {@link + * IllegalStateException}, which propagates out of the status report and aborts it every interval. + * Summing matches what the shared key denotes in that deployment: the totals reached through that + * address. + */ + private static SessionStateForNode mergeNodeStates( + SessionStateForNode first, SessionStateForNode second) { + return new SessionStateForNode( + sumNullable(first.getConnections(), second.getConnections()), + sumNullable(first.getInFlightQueries(), second.getInFlightQueries())); + } + + private static Integer sumNullable(Integer first, Integer second) { + if (first == null) { + return second; + } + return second == null ? first : first + second; } private SessionStateForNode constructSessionStateForNode(Map.Entry entry) { @@ -363,6 +388,35 @@ private long getPeriodicStatusInterval() { return TimeUnit.MILLISECONDS.toSeconds(insightsConfiguration.getStatusEventDelayMillis()); } + /** + * Groups the contact points by host name, which for a hostname contact point now means grouping + * it with itself. + * + *

The field was name to list-of-addresses, and for a name with several A-records that list was + * the interesting part. Contact points are kept unresolved now ({@code SessionBuilder} merges + * them with resolution off), so {@code endPoint.resolve()} hands back one unresolved address per + * contact point and {@code AddressFormatter} renders it as the name again: {@code + * {"db.example.com": ["db.example.com:9042"]}} where it used to be {@code {"db.example.com": + * ["10.0.0.1:9042", "10.0.0.2:9042"]}}. A contact point given as an IP literal is unaffected in + * shape, though see below for what it used to be keyed on. + * + *

Not resolved here to restore the old shape, deliberately. This runs on the admin executor -- + * {@code DefaultSession}'s single-threaded init calls {@code onSessionReady} there -- which is + * the thread the rest of this change went to some length to keep name lookups off (see issue + * #1006), and it is shared with the control connection. Paying a blocking lookup per contact + * point on it to fill in a telemetry field is the wrong trade; the addresses the session actually + * reached are still reported, one of them by {@code #getControlConnectionSocketAddress}. + * + *

Which is also why the key is {@link InetSocketAddress#getHostString()} and not {@code + * getHostName()}. The two agree on everything the paragraph above is about -- an unresolved + * address, and a resolved one built from a name, both carry the label and both return it -- but + * for a resolved address with no label {@code getHostName()} falls through to {@link + * java.net.InetAddress#getHostName()}, a reverse lookup, and that is exactly what {@code + * addContactPoints(new InetSocketAddress(InetAddress.getByAddress(...), port))} produces. + * Grouping on it would put the one blocking lookup this method refuses to make back on the admin + * executor, per such contact point, at each {@code advanced.monitor-reporting} interval, and key + * the map on a reverse-zone answer rather than on anything the operator configured. + */ @VisibleForTesting static Map> getResolvedContactPoints(Set contactPoints) { if (contactPoints == null) { @@ -371,7 +425,7 @@ static Map> getResolvedContactPoints(Set return contactPoints.stream() .collect( Collectors.groupingBy( - InetSocketAddress::getHostName, + InetSocketAddress::getHostString, Collectors.mapping(AddressFormatter::nullSafeToString, Collectors.toList()))); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index dd60a2487fb..be07cf03a24 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -135,6 +135,13 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: int */ CONNECTION_MAX_ORPHAN_REQUESTS("advanced.connection.max-orphan-requests"), + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + * + *

Value-type: int + */ + CONNECTION_MAX_CANDIDATE_ADDRESSES("advanced.connection.max-candidate-addresses"), /** * Whether to log non-fatal errors when the driver tries to open a new connection. * @@ -701,8 +708,29 @@ public enum DefaultDriverOption implements DriverOption { CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"), /** - * Whether to forcibly add original contact points held by MetadataManager to the reconnection - * plan, in case there is no live nodes available according to LBP. Experimental. + * Whether to append the original contact points held by MetadataManager to the reconnection plan, + * after the live nodes reported by the load balancing policy. Defaults to {@code true}. + * + *

This is also the driver's DNS re-resolution path, for the ordinary case. Contact points are + * appended as-is, still unresolved hostnames, and each is expanded to its current DNS IPs at + * connection time through Netty's configured resolver. Metadata nodes, in contrast, hold an + * already-resolved endpoint that is never re-resolved. Keeping this enabled lets + * control-connection reconnects re-resolve the original hostnames and pick up new IPs once the + * live-node plan is exhausted. + * + *

Not the only path, though, and for two kinds of deployment it is barely a path at all: an + * {@code AddressTranslator} that returns a hostname keeps every node's endpoint unresolved, and a + * Cloud (SNI) session builds every endpoint as an {@code SniEndPoint}, which does the same. Both + * re-expand on every attempt whatever this is set to -- and for Cloud the append is skipped + * anyway unless the live-node plan is empty, since {@link + * com.datastax.oss.driver.internal.core.metadata.TopologyMonitor#reresolvesNodeAddresses()} is + * true there. Turning it off then removes only the empty-plan fallback. + * + *

A client-routes session is not a third case, though it looks like one. Its endpoints + * re-expand only for nodes that have a live route; a route-less node falls back to a static, + * already-resolved address, which is why {@code reresolvesNodeAddresses()} answers true there + * only while every known node has a route. Below full coverage the contact points are appended as + * usual and this option is the only re-resolution those nodes get. * *

Value-type: boolean */ @@ -837,7 +865,14 @@ public enum DefaultDriverOption implements DriverOption { * Whether to resolve the addresses passed to `basic.contact-points`. * *

Value-type: boolean + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. */ + @Deprecated RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"), /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index c1a428b3524..bc5bf51c2ba 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException throw new InvalidObjectException("Proxy required"); } + // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the + // defaults map stays complete by carrying its reference.conf value. + @SuppressWarnings("deprecation") protected static void fillWithDriverDefaults(OptionsMap map) { Duration initQueryTimeout = Duration.ofSeconds(5); Duration requestTimeout = Duration.ofSeconds(2); @@ -276,6 +279,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONNECTION_POOL_INIT_BATCH_SIZE, 0); map.put(TypedDriverOption.CONNECTION_MAX_REQUESTS, 1024); map.put(TypedDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, 256); + map.put(TypedDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, 5); map.put(TypedDriverOption.CONNECTION_WARN_INIT_ERROR, true); map.put(TypedDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED, true); map.put(TypedDriverOption.ADVANCED_SHARD_AWARENESS_PORT_LOW, 10000); @@ -369,7 +373,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true); - map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false); + map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true); map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true); map.put(TypedDriverOption.REPREPARE_ENABLED, true); map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index af93e734ef1..fe7888682bc 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -172,6 +172,13 @@ public String toString() { public static final TypedDriverOption CONNECTION_MAX_ORPHAN_REQUESTS = new TypedDriverOption<>( DefaultDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, GenericType.INTEGER); + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + */ + public static final TypedDriverOption CONNECTION_MAX_CANDIDATE_ADDRESSES = + new TypedDriverOption<>( + DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, GenericType.INTEGER); /** Whether to log non-fatal errors when the driver tries to open a new connection. */ public static final TypedDriverOption CONNECTION_WARN_INIT_ERROR = new TypedDriverOption<>(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR, GenericType.BOOLEAN); @@ -600,7 +607,15 @@ public String toString() { public static final TypedDriverOption CONTROL_CONNECTION_AGREEMENT_WARN = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN); - /** Whether to forcibly try original contacts if no live nodes are available */ + /** + * Whether to append the original contact points to the control-connection reconnection plan, + * after the live nodes reported by the load balancing policy (defaults to {@code true}). + * + *

Contact points are appended as-is (unresolved hostnames); each is expanded to all of its + * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The + * append is skipped for topology monitors that re-resolve node addresses themselves (such as the + * cloud/proxy monitors). + */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN); @@ -664,7 +679,16 @@ public String toString() { /** The coalescer reschedule interval. */ public static final TypedDriverOption COALESCER_INTERVAL = new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION); - /** Whether to resolve the addresses passed to `basic.contact-points`. */ + /** + * Whether to resolve the addresses passed to `basic.contact-points`. + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. + */ + @Deprecated public static final TypedDriverOption RESOLVE_CONTACT_POINTS = new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN); /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index 530f2ad38ac..11ef6ca8511 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -18,24 +18,77 @@ package com.datastax.oss.driver.api.core.metadata; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetSocketAddress; import java.net.SocketAddress; /** * Encapsulates the information needed to open connections to a node. * *

By default, the driver assumes plain TCP connections, and this is just a wrapper around an - * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom + * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom * implementation that contains additional information; for example, if the nodes are accessed * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address. */ public interface EndPoint { /** - * Resolves this instance to a socket address. + * Resolves this instance to the socket address connections should be opened to. * *

This will be called each time the driver opens a new connection to the node. The returned * address cannot be null. + * + *

Returning a hostname is fine, and is how multi-address support works. The returned + * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved() + * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to every + * address the name maps to, and each one is tried in turn until a connection succeeds. That is + * what {@code DefaultEndPoint} does for contact points backed by a hostname, so a single + * unreachable IP behind a multi-record name no longer fails the connection. + * + *

One caveat if the name covers several different nodes rather than several addresses + * of one. The driver can bind a connection to the address it reached, so that the node it + * identified there keeps reconnecting to that same address, but only for its own endpoint type -- + * an implementation from outside the driver cannot be bound. Its contact-point connect is still + * spread across the records, so the control connection may identify the node behind the third + * one, while that node's pool keeps the resolver's order and converges on the first. Prefer the + * driver's own {@code DefaultEndPoint} for a name that fronts more than one node; a custom + * implementation is on solid ground for several addresses of a single node, which is the case + * this paragraph is really about. + * + *

Implementations must not resolve names themselves, and must not block. The driver + * calls this from its admin event loop, and it performs the expansion through Netty's configured + * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is + * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with + * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a + * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Callers must not assume the returned address is resolved. It normally is for a node + * discovered from {@code system.peers} (built from that node's physical broadcast RPC address), + * and normally is for the node the control connection is on (bound to the address that connection + * reached). It is not for a node reached through the Cloud SNI proxy, or through a cloud + * private-endpoint client route: there the address is the configured hostname, and {@link + * java.net.InetSocketAddress#getAddress()} returns {@code null}. + * + *

"Normally" is doing work in that sentence, and the control node is where it does most of it. + * Binding the endpoint to the address the connection reached is best effort: it is skipped for an + * endpoint the driver did not build, for one whose {@code resolve()} is not an {@link + * java.net.InetSocketAddress}, and for one that comes back already unresolved -- which is + * what a pipeline that connects through a proxy handler, a disabled resolver, or a custom {@code + * AddressResolverGroup} reporting the name as resolved all produce, and all of which are + * supported. On any of those the control node keeps the contact-point name it was reached + * through. So read the host with {@link java.net.InetSocketAddress#getHostString()}, which yields + * whichever of the two the address carries and never triggers a reverse lookup, rather than + * reaching through {@code getAddress()}. + * + * @apiNote Timeout note: when a name expands to several addresses they are tried in + * sequence, so the worst-case time before the node is declared unreachable is N times a full + * attempt — and an attempt is more than a connect. Each address that accepts the TCP + * connection then runs the init handshake, whose steps each arm their own {@code + * advanced.connection.init-query-timeout}; those add up rather than sharing one deadline. An + * address that stalls after accepting the connection can therefore burn {@code + * advanced.connection.connect-timeout} plus several times {@code + * advanced.connection.init-query-timeout} on its own. In practice DNS round-robin entries + * have only a small number of records, so this is rarely a concern, but it is worth bearing + * in mind when configuring timeouts — note also that session initialization has no overall + * deadline of its own. */ @NonNull SocketAddress resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 8375f0ef30b..409ac5a589f 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad *

Contact points can also be provided statically in the configuration. If both are specified, * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042. * - *

Contrary to the configuration, DNS names with multiple A-records will not be handled here. - * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)} - * before calling this method. Similarly, if you need connect addresses to stay unresolved, make - * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the - * configuration for more explanations). + *

The driver automatically expands any contact point backed by an unresolved hostname to all + * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code + * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its + * IPs on initial connect. This applies equally to hostnames provided here programmatically (build + * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String, + * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address + * passed here (the common case when constructing an {@code InetSocketAddress} directly from a + * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code + * advanced.resolve-contact-points} option is deprecated and has no effect. */ @NonNull public SelfT addContactPoints(@NonNull Collection contactPoints) { @@ -741,6 +745,23 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS * *

For more information, please refer to the DataStax Astra documentation. * + *

A proxy given as a hostname is resolved at connection time, to all of its addresses, + * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the + * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code + * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that + * single address. + * + *

Prefer {@link InetSocketAddress#createUnresolved(String, int)} all the same, and especially + * for a proxy given as an IP address. Whether an address carries a name is read from + * {@code getHostString()}, which falls back to the underlying {@link java.net.InetAddress}'s + * cached host name -- and that field is filled in, on the very instance passed here, the first + * time anything calls {@code getHostName()} on it. The SNI SSL engine does exactly that while + * building an engine, unless reverse-lookup SANs are turned off. So an IP that has a {@code PTR} + * record can acquire a name mid-session, after which the driver treats that name as the proxy: + * the endpoints it builds from then on compare unequal to the earlier ones, report metrics under + * a different prefix, and connect to wherever that name resolves. An unresolved address is never + * subject to this, and is what the secure connect bundle produces. + * * @param cloudProxyAddress The address of the Cloud proxy to use. * @see Server Name Indication */ @@ -957,11 +978,15 @@ protected final CompletionStage buildDefaultSessionAsync() { programmaticArguments = programmaticArgumentsBuilder.build(); } - boolean resolveAddresses = - defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false); - + // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved + // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory. + // The value is still read, only to tell someone who set it that it no longer does anything. + // Note this tests the value rather than isDefined(): unlike the deprecated options warned + // about in DefaultDriverContext, this one ships uncommented in reference.conf, so it is + // always defined. + warnIfResolveContactPointsRequested(defaultConfig); Set contactPoints = - ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses); + ContactPoints.merge(programmaticContactPoints, configContactPoints, false); if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) { keyspace = @@ -989,6 +1014,24 @@ private boolean anyProfileHasDatacenterDefined(DriverConfig driverConfig) { return false; } + /** + * Tells anyone who turned {@code advanced.resolve-contact-points} on that it no longer does + * anything, so the behaviour they configured does not disappear in silence. + */ + @SuppressWarnings("deprecation") + private static void warnIfResolveContactPointsRequested(DriverExecutionProfile defaultConfig) { + if (defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false)) { + LOG.warn( + "Option {} is deprecated and no longer has any effect. Contact points given in the" + + " configuration are now always kept as unresolved hostnames and expanded to all of" + + " their addresses at connection time, so a name that resolves to several nodes is" + + " one Node in the driver's metadata rather than one per address. Note that this" + + " never applied to contact points passed to addContactPoints(): those are used" + + " exactly as supplied, and an already-resolved address stays bound to that one IP.", + DefaultDriverOption.RESOLVE_CONTACT_POINTS.getPath()); + } + } + /** * Returns URL based on the configUrl setting. If the configUrl has no protocol provided, the * method will fallback to file:// protocol and return URL that has file protocol specified. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslator.java index bee22dc5335..32bfe1687f3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslator.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslator.java @@ -81,6 +81,25 @@ public Ec2MultiRegionAddressTranslator( this.ctx = ctx; } + /** + * {@inheritDoc} + * + *

The domain name the PTR record gives is returned {@linkplain InetSocketAddress#isUnresolved + * () unresolved}, so that {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} + * expands it per connect and can try every address it maps to. Only the forward lookup + * moves; the reverse one below still happens here, because it is the whole point of this class. + * + *

Resolving forward here would keep whichever address came back first and no other would ever + * be tried, since the resolver reports an already-resolved address as nothing to do and the + * expansion is skipped. Same defect, and same fix, as {@link FixedHostNameAddressTranslator} -- + * the two get there differently ({@code InetAddress.getByName} rather than {@code new + * InetSocketAddress(String, int)}) but arrive at the same resolved address. + * + *

The forward lookup is still performed here, and its result discarded: it is what + * tells this method that the name it is about to hand over is usable, while it still holds the + * address to fall back on if it is not. A name that does not resolve is answered with the + * original {@code socketAddress}, exactly as it was before the lookup moved. + */ @NonNull @Override public InetSocketAddress translate(@NonNull InetSocketAddress socketAddress) { @@ -95,9 +114,37 @@ public InetSocketAddress translate(@NonNull InetSocketAddress socketAddress) { return socketAddress; } - InetAddress translatedAddress = InetAddress.getByName(domainName); - LOG.debug("[{}] Resolved {} to {}", logPrefix, address, translatedAddress); - return new InetSocketAddress(translatedAddress, socketAddress.getPort()); + // Resolved and thrown away, on purpose. What moves into the connection attempt is the + // *choice* of address -- so that every A-record gets a turn -- not the question of whether + // the name resolves at all, and that question is this method's to answer: it is the one + // place that still holds the working address the PTR record was found from. Returning an + // unresolvable name unchecked strands the node for good, since the connect layer has nothing + // to fall back to and every refresh re-derives the same name from the same PTR record. A PTR + // whose target has no A record is not exotic -- private DNS switched off on the VPC, or + // split-horizon DNS that answers the reverse zone but not the forward one -- and before the + // lookup moved, the UnknownHostException it throws is what reached the catch below and handed + // back the node's raw broadcast address. + // + // Exactly that question and no more. A PTR record that is merely *stale* -- naming a host + // that still resolves, because it is the instance that replaced this one -- passes, since the + // addresses come back only to be counted and are never compared against the one this node + // answered on. And the question is answered by the JVM's resolver, while the connect goes + // through whichever AddressResolverGroup NettyOptions installed: a deployment that points + // Netty at a private zone the JVM cannot see, or hands resolution to a pipeline handler + // entirely, would have a name rejected here that the connect could have used. Both are + // tracked in https://github.com/scylladb/java-driver/issues/1010, and neither is fixable from + // here -- this method is synchronous and knows nothing about Netty. + // + // Cheap where it sits: the JVM caches forward lookups (networkaddress.cache.ttl), and this + // method already pays for an uncached JNDI reverse lookup on the same call. + InetAddress[] forward = InetAddress.getAllByName(domainName); + LOG.debug( + "[{}] Resolved {} to {}, which maps to {} address(es)", + logPrefix, + address, + domainName, + forward.length); + return InetSocketAddress.createUnresolved(domainName, socketAddress.getPort()); } catch (Exception e) { Loggers.warnWithException( LOG, "[{}] Error resolving {}, returning it as-is", logPrefix, address, e); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java index 0ee1e22a7ac..70db25d2a5c 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java @@ -53,12 +53,24 @@ public FixedHostNameAddressTranslator(@NonNull DriverContext context) { context.getConfig().getDefaultProfile().getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME); } + /** + * {@inheritDoc} + * + *

The advertised host name is returned {@linkplain InetSocketAddress#isUnresolved() + * unresolved}, so that {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} + * expands it per connect and can try every address it maps to. Resolving it here -- which {@code + * new InetSocketAddress(String, int)} does eagerly -- would freeze the whole cluster on whichever + * address the JDK happened to return first, and no other would ever be tried: the resolver + * reports an already-resolved address as nothing to do, so the expansion is skipped entirely. + * That matters precisely for the deployment this translator is for, where one name fronts a proxy + * or load balancer that is itself typically several addresses. + */ @NonNull @Override public InetSocketAddress translate(@NonNull InetSocketAddress address) { final int port = address.getPort(); LOG.debug("[{}] Resolved {}:{} to {}:{}", logPrefix, address, port, advertisedHostname, port); - return new InetSocketAddress(advertisedHostname, port); + return InetSocketAddress.createUnresolved(advertisedHostname, port); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java index 5078428c21a..02496d68ae1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java @@ -30,8 +30,9 @@ import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.request.Query; +import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.query.QueryOptions; -import com.datastax.oss.protocol.internal.response.Result; +import com.datastax.oss.protocol.internal.response.Ready; import com.datastax.oss.protocol.internal.response.result.Prepared; import com.datastax.oss.protocol.internal.response.result.Rows; import io.netty.util.concurrent.Future; @@ -67,6 +68,24 @@ public static AdminRequestHandler call( com.datastax.oss.protocol.internal.response.result.Void.class); } + /** + * Registers this connection for the given protocol events, as {@link + * com.datastax.oss.protocol.internal.request.Register REGISTER} used to be sent as the last step + * of protocol initialization. + */ + public static AdminRequestHandler register( + DriverChannel channel, List eventTypes, Duration timeout, String logPrefix) { + return new AdminRequestHandler<>( + channel, + true, + new Register(eventTypes), + Frame.NO_PAYLOAD, + timeout, + logPrefix, + "register for events " + eventTypes, + Ready.class); + } + public static AdminRequestHandler query( DriverChannel channel, String query, @@ -98,7 +117,7 @@ public static AdminRequestHandler query( private final Duration timeout; private final String logPrefix; private final String debugString; - private final Class expectedResponseType; + private final Class expectedResponseType; protected final CompletableFuture result = new CompletableFuture<>(); // This is only ever accessed on the channel's event loop, so it doesn't need to be volatile @@ -112,7 +131,7 @@ protected AdminRequestHandler( Duration timeout, String logPrefix, String debugString, - Class expectedResponseType) { + Class expectedResponseType) { this.channel = channel; this.shouldPreAcquireId = shouldPreAcquireId; this.message = message; @@ -190,8 +209,9 @@ public void onResponse(Frame responseFrame) { @SuppressWarnings("unchecked") ResultT result = (ResultT) ByteBuffer.wrap(prepared.preparedQueryId); setFinalResult(result); - } else if (expectedResponseType - == com.datastax.oss.protocol.internal.response.result.Void.class) { + } else if (expectedResponseType == com.datastax.oss.protocol.internal.response.result.Void.class + || expectedResponseType == Ready.class) { + // Neither carries a payload: a schema change or a REGISTER acknowledgement. setFinalResult(null); } else { setFinalError(new AssertionError("Unhandled response type" + expectedResponseType)); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 35190afa3f4..b241298650d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -24,29 +24,40 @@ package com.datastax.oss.driver.internal.core.channel; import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.InvalidKeyspaceException; import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.context.DriverContext; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeShardingInfo; import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; +import com.datastax.oss.driver.internal.core.adminrequest.UnexpectedResponseException; import com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverConfig; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; import com.datastax.oss.driver.internal.core.protocol.FrameDecoder; import com.datastax.oss.driver.internal.core.protocol.FrameEncoder; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.driver.internal.core.util.ProtocolUtils; +import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.protocol.internal.Message; +import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -54,18 +65,35 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoop; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.Timeout; +import io.netty.util.concurrent.Future; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import net.jcip.annotations.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +155,24 @@ public static int effectiveMaxOrphanRequests( private final String logPrefix; protected final InternalDriverContext context; + /** + * Guards the one-time warning in {@link #newBootstrap()}. Per factory rather than per JVM: what + * it reports is a property of this session's {@link NettyOptions}, and the message names the + * session, so a JVM-wide latch would report the first offender and silence every one after it. + */ + private final AtomicBoolean loggedHandlerWarning = new AtomicBoolean(); + + private final AtomicBoolean loggedGroupWarning = new AtomicBoolean(); + + /** Guards the one-time warning in {@link #warnAboutPassThrough}, on the same terms. */ + private final AtomicBoolean loggedPassThroughWarning = new AtomicBoolean(); + + /** + * Randomizes the order in which a name's expanded addresses are tried (see {@link + * #shuffleAndLimit}). Injectable so tests can seed it and observe a deterministic order. + */ + @VisibleForTesting Random random = new Random(); + /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -145,6 +191,7 @@ public ChannelFactory(InternalDriverContext context) { this.context = context; DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); + if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); this.protocolVersion = context.getProtocolVersionRegistry().fromName(versionName); @@ -181,7 +228,7 @@ public CompletionStage connect(Node node, DriverChannelOptions op } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater); + return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater, isIdentified(node)); } public CompletionStage connect( @@ -192,7 +239,31 @@ public CompletionStage connect( } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), node.getShardingInfo(), shardId, options, nodeMetricUpdater); + return connect( + node.getEndPoint(), + node.getShardingInfo(), + shardId, + options, + nodeMetricUpdater, + isIdentified(node)); + } + + /** + * Whether we know which node we are connecting to, as opposed to merely which address to + * try. Both {@link #spreadAcrossAddresses} and {@link #sameServerAtEveryAddress} turn on it, + * because an unidentified contact-point name may expand to addresses of different nodes, + * while every address of an identified node is that same node. + * + *

{@link Node#getHostId()} is null exactly for a contact point, and stays null for the life of + * that instance: {@code MetadataManager.registerNode} mints a fresh {@code DefaultNode} from each + * {@code system.local}/{@code system.peers} row rather than back-filling the contact point it was + * reached through, and those ephemeral contact-point nodes are never added to metadata. So this + * is not a state a contact point grows out of once the driver has read host ids -- the driver + * simply stops using that instance for anything except the reconnection fallback, which keeps + * handing it back. + */ + private static boolean isIdentified(Node node) { + return node.getHostId() != null; } @VisibleForTesting @@ -202,11 +273,23 @@ CompletionStage connect( Integer shardId, DriverChannelOptions options, NodeMetricUpdater nodeMetricUpdater) { + // A bare endpoint carries no host id, so this matches the contact-point case (see + // isIdentified()). + return connect(endPoint, shardingInfo, shardId, options, nodeMetricUpdater, false); + } + + @VisibleForTesting + CompletionStage connect( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + boolean nodeIsIdentified) { CompletableFuture resultFuture = new CompletableFuture<>(); ProtocolVersion currentVersion; boolean isNegotiating; - List attemptedVersions = new CopyOnWriteArrayList<>(); if (this.protocolVersion != null) { currentVersion = protocolVersion; isNegotiating = false; @@ -223,7 +306,7 @@ CompletionStage connect( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, + nodeIsIdentified, resultFuture); return resultFuture; } @@ -236,121 +319,1914 @@ private void connect( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, - List attemptedVersions, + boolean nodeIsIdentified, CompletableFuture resultFuture) { - SocketAddress resolvedAddress; + // Built once per connect() rather than once per candidate: it is the only handle on the Netty + // AddressResolverGroup (see resolveCandidates()), and it means the user's + // afterBootstrapInitialized() hook runs once per logical connection instead of once per address + // attempt. Each attempt gets its own clone() with its own handler. + // + // The event loop is likewise picked once per connect() and shared by name resolution and the + // channel itself (the per-attempt clones are bound to it, see connectToAddress()). Advancing + // the group's round-robin chooser exactly once per connect keeps channels evenly distributed: + // taking one loop for resolution and letting Bootstrap.connect() take another would advance + // the chooser twice per connect, parking all channels on half the loops with the default + // power-of-two chooser. It also mirrors what Netty itself does with an unresolved address: + // Bootstrap resolves on the connecting channel's own event loop. + Bootstrap baseBootstrap; + EventLoop eventLoop; try { - resolvedAddress = endPoint.resolve(); - } catch (Exception e) { + baseBootstrap = newBootstrap(); + eventLoop = context.getNettyOptions().ioEventLoopGroup().next(); + } catch (Throwable e) { resultFuture.completeExceptionally(e); return; } - NettyOptions nettyOptions = context.getNettyOptions(); + // EndPoint.resolve() is contractually non-blocking and performs no name resolution, so it is + // safe to call here even though connect() runs on the admin event loop for control-connection + // reconnects. Everything a name needs to become connectable happens in resolveCandidates(). + SocketAddress address; + try { + address = endPoint.resolve(); + } catch (Throwable e) { + resultFuture.completeExceptionally(e); + return; + } + if (address == null) { + // EndPoint.resolve() is contractually non-null; fail fast instead of NPE-ing inside an + // event-loop task later, which would leave resultFuture hanging (see resolveCandidates()). + resultFuture.completeExceptionally( + new IllegalArgumentException("EndPoint.resolve() returned null: " + endPoint)); + return; + } + // Guarded for the same reason resolve() is: addressesAreInterchangeable() calls through to + // PinnableEndPoint.addressesAreInterchangeable(), another method the endpoint implementation + // supplies and can therefore throw from. As a bare argument to resolveCandidates() it would + // escape connect() synchronously, and nothing upstream would complete resultFuture: + // ControlConnection.reconnect() does not wrap its connect() call, and the recursive ones run + // inside a whenCompleteAsync callback with no catch, so the throwable would be swallowed and + // the attempt left hanging with Reconnection stuck in ATTEMPT_IN_PROGRESS. + // + // Throwable, not Exception, and likewise for the two guards above: an endpoint supplied by + // someone else can fail with an Error just as easily as with an exception -- a + // NoClassDefFoundError or ExceptionInInitializerError out of lazy class initialization in a + // shaded or OSGi deployment, an AssertionError under -ea -- and a hang is the outcome either + // way. Every other guard in this class that owns a future's completion already catches + // Throwable. + boolean interchangeable; + try { + interchangeable = addressesAreInterchangeable(endPoint, address); + } catch (Throwable e) { + resultFuture.completeExceptionally(e); + return; + } + + // Two questions over the same two facts. Not each other's negation: a connect can be both + // (an identified node behind an SNI proxy) or neither (an identified node on a plain name). + boolean spreadAcrossAddresses = spreadAcrossAddresses(nodeIsIdentified, interchangeable); + boolean sameServerAtEveryAddress = sameServerAtEveryAddress(nodeIsIdentified, interchangeable); + + resolveCandidates(baseBootstrap, address, eventLoop, spreadAcrossAddresses) + .whenComplete( + (candidates, error) -> { + if (error != null) { + Throwable cause = + (error instanceof CompletionException && error.getCause() != null) + ? error.getCause() + : error; + resultFuture.completeExceptionally(cause); + return; + } + tryNextCandidate( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + sameServerAtEveryAddress, + resultFuture, + candidates, + 0, + new ArrayList<>()); + }); + } + + /** + * Builds the {@link Bootstrap} shared by every connection attempt of a single {@code connect()} + * call, including the user's {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} hook. Per + * attempt, {@link #connectToAddress} takes a {@link + * Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it bound to the event loop the connect + * picked, and installs its own handler; the copy carries the resolver configuration over. The + * base bootstrap itself keeps the full I/O group, so the hook observes the same group as always. + */ + private Bootstrap newBootstrap() { + NettyOptions nettyOptions = context.getNettyOptions(); Bootstrap bootstrap = new Bootstrap() .group(nettyOptions.ioEventLoopGroup()) .channel(nettyOptions.channelClass()) - .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()) - .handler( - initializer(endPoint, currentVersion, options, nodeMetricUpdater, resultFuture)); - + .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()); nettyOptions.afterBootstrapInitialized(bootstrap); + if (bootstrap.config().handler() != null && loggedHandlerWarning.compareAndSet(false, true)) { + LOG.warn( + "[{}] NettyOptions.afterBootstrapInitialized() installed a channel handler on the" + + " bootstrap; it will be replaced by the driver's own handler. Use" + + " NettyOptions.afterChannelInitialized() to customize the pipeline instead.", + logPrefix); + } + // Same shape as the handler above, and for the same reason: connectToAddress() clones this + // bootstrap with clone(eventLoop), which assigns the group unconditionally, so a group the + // hook set here is dropped and the driver's own ioEventLoopGroup is used instead. Silently + // moving a deployment's I/O back onto the driver's threads is exactly the kind of thing that + // is noticed months later, so say it once. + if (bootstrap.config().group() != nettyOptions.ioEventLoopGroup() + && loggedGroupWarning.compareAndSet(false, true)) { + LOG.warn( + "[{}] NettyOptions.afterBootstrapInitialized() replaced the bootstrap's event loop" + + " group; it will be ignored, because each connection attempt is bound to an event" + + " loop picked from NettyOptions.ioEventLoopGroup(). Override ioEventLoopGroup() to" + + " run driver I/O on your own threads.", + logPrefix); + } + return bootstrap; + } - ChannelFuture connectFuture; - if (shardId == null || shardingInfo == null) { - if (shardId != null) { - LOG.debug( - "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", - shardId, - endPoint); - } - connectFuture = bootstrap.connect(resolvedAddress); - } else { - int localPort = - PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); - if (localPort == -1) { - LOG.warn( - "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", - shardId, - endPoint); - connectFuture = bootstrap.connect(resolvedAddress); - } else { - connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + /** + * Turns the address an {@link EndPoint} denotes into the concrete, connectable addresses to try, + * expanding it to all the addresses it maps to when it is a name. + * + *

Expansion goes through the bootstrap's Netty {@link AddressResolverGroup} rather than a + * direct {@code InetAddress.getAllByName()} call, so a custom resolver installed via {@link + * NettyOptions#afterBootstrapInitialized(Bootstrap)} is honoured — that is the resolver an + * unresolved address would have reached had it been handed straight to {@code + * Bootstrap.connect()}, as it was before multi-address support. This is also why endpoints are + * forbidden from resolving names themselves (see {@link EndPoint#resolve()}): doing it here is + * the only way to keep that configuration point working, and the only way to keep {@code + * resolve()} non-blocking. + * + *

Whether an address needs resolving at all is the resolver's decision, not ours: exactly as + * in {@code Bootstrap#doResolveAndConnect0}, the address is passed through untouched only when + * the resolver says it does not {@linkplain AddressResolver#isSupported support} it (e.g. {@link + * io.netty.channel.local.LocalAddress}) or that it {@linkplain AddressResolver#isResolved is + * already resolved}. Both are overridable, and a custom resolver may well report an + * already-resolved address as unresolved in order to redirect it — Netty consulted it either way, + * so a pre-check here on {@code InetSocketAddress#isUnresolved()} would silently take that + * configuration point away for every connect to an already-resolved node, which is to say for + * almost every connect. A null group means the user called {@link Bootstrap#disableResolver()}, + * which is likewise respected. + * + *

On the two branches where nothing is going to resolve the address -- no resolver at + * all, or one that declines it -- an unresolved IP literal is materialized locally instead of + * being failed; see {@link #materializeLiteral}. A host name still fails there, with a message + * naming which of the two put it in that position. The third pass-through branch is different in + * kind: a resolver that reports the address already resolved has claimed it, so the address goes + * out untouched and only a warning is logged (see {@link #warnAboutPassThrough}). + * + *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, + * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is + * the pre-existing behaviour of handing an unresolved address to {@code Bootstrap.connect()}, and + * it is an I/O loop, never the admin loop that {@code connect()} is called from. Deployments that + * need non-blocking resolution can now install {@code DnsAddressResolverGroup} and have it take + * effect. + */ + private CompletionStage> resolveCandidates( + Bootstrap bootstrap, + SocketAddress address, + EventLoop eventLoop, + boolean spreadAcrossAddresses) { + + AddressResolverGroup resolverGroup = bootstrap.config().resolver(); + if (resolverGroup == null) { + // Bootstrap.disableResolver(): the user wants the address passed through as-is, which only + // works if it is usable as-is -- or can be made so without resolving anything. + SocketAddress literal = materializeLiteral(address); + if (literal != null) { + return CompletableFuture.completedFuture(Collections.singletonList(literal)); } + IllegalStateException unusable = + unusableWithoutResolution( + address, + "the bootstrap has name resolution disabled", + "Either remove Bootstrap.disableResolver() from" + + " NettyOptions.afterBootstrapInitialized(), or supply an already-resolved" + + " address."); + return (unusable != null) + ? CompletableFutures.failedFuture(unusable) + : CompletableFuture.completedFuture(Collections.singletonList(address)); } - connectFuture.addListener( - cf -> { - if (connectFuture.isSuccess()) { - Channel channel = connectFuture.channel(); - DriverChannel driverChannel = - new DriverChannel(endPoint, channel, context.getWriteCoalescer(), currentVersion); - // If this is the first successful connection, remember the protocol version and - // cluster name for future connections. - if (isNegotiating) { - ChannelFactory.this.protocolVersion = currentVersion; - } - if (ChannelFactory.this.clusterName == null) { - ChannelFactory.this.clusterName = driverChannel.getClusterName(); - } - Map> supportedOptions = driverChannel.getOptions(); - if (ChannelFactory.this.productType == null && supportedOptions != null) { - List productTypes = supportedOptions.get("PRODUCT_TYPE"); - String productType = - productTypes != null && !productTypes.isEmpty() - ? productTypes.get(0) - : UNKNOWN_PRODUCT_TYPE; - ChannelFactory.this.productType = productType; - DriverConfig driverConfig = context.getConfig(); - if (driverConfig instanceof TypesafeDriverConfig - && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { - ((TypesafeDriverConfig) driverConfig) - .overrideDefaults( - ImmutableMap.of( - DefaultDriverOption.REQUEST_CONSISTENCY, - ConsistencyLevel.LOCAL_QUORUM.name())); + // The supplied event loop is the same one the channel will be registered on (see connect()), + // which is what Netty itself does with an unresolved address: Bootstrap resolves on the + // connecting channel's own event loop. Its transport also matches the channel class, which + // matters because DnsAddressResolverGroup registers a datagram channel on the executor it + // resolves for. + CompletableFuture> result = new CompletableFuture<>(); + // Every path below must complete `result`: nothing at this stage has a timeout, so a task or + // listener that dies with the future still pending (Netty swallows their throwables, it only + // logs them) would hang the connect attempt -- and with it control-connection init or a pool + // reconnect -- forever. Hence the blanket catches around the task body, the listener body, and + // the execute() call itself (which throws RejectedExecutionException while shutting down). + try { + eventLoop.execute( + () -> { + try { + AddressResolver resolver = + resolverGroup.getResolver(eventLoop); + boolean unsupported = !resolver.isSupported(address); + if (unsupported || resolver.isResolved(address)) { + // Nothing for the resolver to do; same short-circuit as + // Bootstrap#doResolveAndConnect0. The two halves are not the same situation, + // though, and are not treated the same. + // + // An address the resolver *declines* is in the same position as one with no + // resolver at all: nothing has taken responsibility for it and nothing downstream + // will resolve it. So it gets the same check, and the same rescue for an IP + // literal, which needs no name service to begin with. + // + // An address the resolver reports as *already resolved* is a claim, and the claim + // is honoured even when the address plainly is not resolved. That combination is + // not a broken resolver, it is NoopAddressResolverGroup: Netty's documented way of + // saying "leave the name alone, something in the pipeline will deal with it", + // which is exactly what a ProxyHandler installed through + // NettyOptions#afterChannelInitialized(Channel) does -- it intercepts the connect + // and sends the name on to the proxy instead of to a socket. Netty itself hands + // such an address straight to doConnect(), so refusing it here would turn a + // supported configuration into a hard failure of every connect for the whole + // session. It passes through, with one warning for the deployment that arrived + // here by accident rather than on purpose. + if (!unsupported) { + warnAboutPassThrough(address, resolver); + result.complete(Collections.singletonList(address)); + return; + } + SocketAddress literal = materializeLiteral(address); + if (literal != null) { + result.complete(Collections.singletonList(literal)); + return; + } + IllegalStateException unusable = + unusableWithoutResolution( + address, + "the configured resolver does not support this address", + "Either install a resolver that supports it in" + + " NettyOptions.afterBootstrapInitialized(), or supply an" + + " already-resolved address."); + if (unusable != null) { + result.completeExceptionally(unusable); + } else { + result.complete(Collections.singletonList(address)); + } + return; } + resolver + .resolveAll(address) + .addListener( + (Future> future) -> { + try { + if (!future.isSuccess()) { + result.completeExceptionally(future.cause()); + return; + } + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + if (addresses == null || addresses.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + "Resolver returned no address for " + address)); + return; + } + List connectable = + dropUnresolved(address, reattachHostnames(address, addresses)); + if (connectable.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + String.format( + "Cannot connect to %s: the configured resolver (%s) " + + "expanded it to %d address(es) and every one of them " + + "is still unresolved, so nothing will resolve them.", + address, resolver.getClass().getName(), addresses.size()))); + return; + } + result.complete(shuffleAndLimit(connectable, spreadAcrossAddresses)); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); } - resultFuture.complete(driverChannel); - } else { - Throwable error = connectFuture.cause(); - if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { - attemptedVersions.add(currentVersion); - Optional downgraded = - context.getProtocolVersionRegistry().downgrade(currentVersion); - if (downgraded.isPresent()) { + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + return result; + } + + /** + * Says once that a resolver reported an address it plainly has not resolved, and that {@link + * #resolveCandidates} took it at its word. + * + *

The claim is honoured because it is a supported thing to say. {@code + * NoopAddressResolverGroup} reports every address resolved, and it is Netty's own way to hand + * name resolution to something in the pipeline -- a {@code ProxyHandler} added through {@link + * NettyOptions#afterChannelInitialized(io.netty.channel.Channel)}, which intercepts the connect + * and sends the unresolved name to the proxy. Netty's {@code Bootstrap#doResolveAndConnect0} + * short-circuits on exactly this and calls {@code doConnect()} with the address untouched, so + * that deployment worked before multi-address support and has to keep working. It is also the one + * path that leaves {@link PinnableEndPoint#pinTo} an unresolved address, which is why that method + * documents refusing one. + * + *

But the same claim is what a resolver whose {@code isResolved()} is simply wrong makes, and + * that deployment gets no resolution and no proxy either -- just {@code + * UnresolvedAddressException} out of {@code doConnect}, naming neither the address nor the + * reason. Hence the warning: it costs the intentional case one log line and gives the accidental + * one the only diagnosis it will get. Only for an address that really is unresolved, since a + * resolved one passing through is unremarkable. + */ + private void warnAboutPassThrough(SocketAddress address, AddressResolver resolver) { + if (address instanceof InetSocketAddress + && ((InetSocketAddress) address).isUnresolved() + && loggedPassThroughWarning.compareAndSet(false, true)) { + LOG.warn( + "[{}] {} reports {} as already resolved, so it is being connected to unresolved. That is" + + " what NoopAddressResolverGroup does when something in the pipeline resolves the" + + " name instead (a ProxyHandler added in NettyOptions.afterChannelInitialized()); if" + + " nothing does, the connect will fail with UnresolvedAddressException. This message" + + " is logged once.", + logPrefix, + resolver.getClass().getName(), + address); + } + } + + /** + * The failure to report when {@link #resolveCandidates} is about to pass an address through + * without resolving it, or {@code null} if passing it through is fine. + * + *

{@link #connectToAddress} hands the candidate to a bootstrap clone with {@link + * Bootstrap#disableResolver()}, so an address that is still unresolved by the time it gets there + * cannot connect: Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}, + * naming neither the address nor the reason nothing resolved it. That is a hard failure of every + * connection attempt for the whole session, and it is worth a message that says which endpoint + * and which configuration produced it -- the endpoints most likely to hit it (SNI, client routes) + * hand out unresolved addresses by design, and contact-point hostnames are now always kept + * unresolved. + * + *

Deliberately not a general {@code isUnresolved()} pre-check on every path: see {@link + * #resolveCandidates}'s javadoc for why an address the resolver merely declines to touch must + * still go through. This fires only where nothing downstream will resolve it either -- as does + * {@link #dropUnresolved}, which applies the same reasoning to what {@code resolveAll} returns. + * + *

Its callers try {@link #materializeLiteral} first, so this is reached only by an address + * that genuinely needs a name service. "Supply an already-resolved address" is therefore always + * advice about a host name. + * + * @param why what put the address in this position, in a clause that reads after "it is an + * unresolved address and". + * @param fix what the operator should do about it. Each caller supplies its own: the two + * situations that reach here are diagnosed differently, and naming the wrong one sends the + * operator looking for configuration nobody wrote. + */ + private static IllegalStateException unusableWithoutResolution( + SocketAddress address, String why, String fix) { + if (!(address instanceof InetSocketAddress) || !((InetSocketAddress) address).isUnresolved()) { + return null; + } + return new IllegalStateException( + String.format( + "Cannot connect to %s: it is an unresolved address and %s, so nothing will resolve it. %s", + address, why, fix)); + } + + /** + * The address as something connectable when nothing is going to resolve it, or {@code null} if it + * is not an unresolved IP literal. + * + *

A literal needs no name service, so an endpoint that holds one has no business failing on a + * path where resolution is unavailable -- and endpoints now hold one routinely, contact points + * being kept unresolved whatever they were written as (see {@code + * SessionBuilder#addContactPoint}). Before that, {@code 127.0.0.1:9042} arrived here already + * resolved and {@link Bootstrap#disableResolver()} worked with it; this keeps that true. + * + *

Deliberately not a general pre-check. It is applied only where {@link #resolveCandidates} is + * already committed to passing the address through unresolved, never before an enabled resolver + * has been consulted: a custom resolver is entitled to redirect a literal, exactly as it is + * entitled to redirect a name, and testing for one earlier would take that away. + * + *

Consults no name service, which is what makes it safe on both call sites -- one runs on a + * Netty I/O loop, the other on whatever thread called {@code connect()}, the admin loop for the + * control connection. {@link InetAddress#getByName} goes to DNS only for a name, and {@link + * AddressUtils#carriesName} has just established there is none. The one exception is a literal + * carrying a named IPv6 zone ({@code fe80::1%eth0}): the JDK turns the name into a scope + * id through {@code NetworkInterface}, which is a syscall rather than a lookup. Bounded and + * local, so it is accepted rather than special-cased. + * + *

Two spellings {@link AddressUtils#carriesName} calls literals do not survive {@code + * getByName}, and both fall through to the caller's diagnostic -- which then gives advice about + * host names, the one thing {@link #unusableWithoutResolution} promises it is always about: + * + *

+ * + *

The literal is re-attached as the address's host-name label rather than left off, so that + * {@code getHostName()} stays a field read answering what the operator configured. A nameless + * address sends {@code DefaultSslEngineFactory} to a reverse lookup on an event loop and has it + * validate the certificate against a PTR record -- the same hazard {@link #reattachHostname}'s + * literal branch exists to prevent, and the reason the label goes on here rather than being left + * to that method, whose byte-matching re-derives through {@link AddressUtils#parseLiteral} what + * is known here by construction (and which drops a zone rather than carrying it). The label is + * the spelling that was configured, less the brackets of the URI form -- {@link + * InetAddress#getByAddress(String, byte[])} strips those from any host name it is handed. A + * non-canonically written literal then makes {@link AddressUtils#carriesName} report {@code true} + * for the result: the same imprecision that method already documents, and nothing re-labels a + * candidate twice. + */ + @VisibleForTesting + static SocketAddress materializeLiteral(SocketAddress address) { + if (!(address instanceof InetSocketAddress)) { + return null; + } + InetSocketAddress inet = (InetSocketAddress) address; + if (!inet.isUnresolved() || AddressUtils.carriesName(inet)) { + return null; + } + String literal = inet.getHostString(); + try { + return new InetSocketAddress( + AddressUtils.withHostName(literal, InetAddress.getByName(literal)), inet.getPort()); + } catch (UnknownHostException notAfterAll) { + // carriesName() and getByName() disagreeing about what a literal is: fall through to the + // diagnostic the caller was about to raise, which says more than a bare parse failure. + return null; + } + } + + /** Applies {@link #reattachHostname} to every expanded candidate. */ + private static List reattachHostnames( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + result.add(reattachHostname(original, candidate)); + } + return result; + } + + /** + * Drops the candidates a resolver returned still unresolved, keeping the order of the rest. + * + *

{@code resolveAll} is contracted to return resolved addresses, but a custom resolver that + * rewrites what it is given -- which {@link #resolveCandidates} deliberately supports -- may hand + * back one that is not. Such a candidate cannot connect: {@link #connectToAddress} uses a + * bootstrap clone with {@link Bootstrap#disableResolver()}, so nothing downstream will resolve it + * either, and Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}. This + * is the same reasoning as {@link #unusableWithoutResolution}, applied where the addresses come + * from the resolver itself; dropping them here rather than after the cap means the cap counts + * only addresses that can actually be tried. The caller reports the case where nothing is left, + * which is the one that fails every connection attempt for the whole session. + */ + private List dropUnresolved( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + if (candidate instanceof InetSocketAddress + && ((InetSocketAddress) candidate).isUnresolved()) { + LOG.debug( + "[{}] Resolver returned {} for {} but it is still unresolved, skipping it", + logPrefix, + candidate, + original); + } else { + result.add(candidate); + } + } + return result; + } + + /** + * Re-attaches the {@code original} address's host name to one of the resolved candidates it + * expanded to, whatever name that candidate carries. + * + *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es + * they return, so this is a no-op for them. A custom resolver, however, may build its results + * from raw address bytes, or label them with a canonical/CNAME name of its own. The channel's + * pinned endpoint is built from the candidate (see {@link PinnableEndPoint}), and it is what + * {@code DefaultSslEngineFactory} and {@code SniSslEngineFactory} derive the SSL peer host from, + * inside the channel initializer. So whatever name the candidate carries is the name TLS hostname + * verification checks the server certificate against, and the only name that may be is the one + * the user configured: with a nameless address, {@code InetSocketAddress#getHostName()} + * additionally triggers a blocking reverse-DNS lookup on the event loop and validation falls back + * to the IP or the PTR record, and with a resolver-supplied label it validates a name the + * operator never chose. Hence the queried name always wins here; before multi-address support the + * initializer kept the original endpoint and Netty resolved only the TCP destination, which had + * the same effect. + * + *

Re-attaching changes nothing else: {@link AddressUtils#withHostName} performs no lookup, the + * TCP connect target is the same IP, and a resolved {@link InetSocketAddress}'s equality ignores + * host names, so pinning and the pin-equality shortcuts are unaffected. A scoped IPv6 candidate + * keeps its zone. + * + *

An IP literal gets its own literal re-attached, and only when the resolver handed + * back that very address. Leaving the candidate nameless there would not be neutral: a nameless + * address is exactly what {@code InetSocketAddress#getHostName()} answers with a blocking reverse + * lookup, so {@code DefaultSslEngineFactory} would validate the certificate against a PTR record + * instead of the literal the operator configured, on a Netty I/O loop — where before, contact + * points were kept unresolved and the literal came back with no lookup at all. Labelling with the + * literal keeps {@code getHostName()} a field read that answers the literal, which is what it + * answered before. (A non-canonically written IPv6 literal then makes {@link + * AddressUtils#carriesName} report {@code true} for the labelled candidate: the same imprecision + * that method already documents, and nothing re-labels a candidate twice.) + * + *

A candidate the resolver redirected to a different IP is left alone: labelling it + * with the literal form of the one we asked for would invent a name that resolves to something + * else. + * + *

A resolved original is treated exactly like an unresolved one, and reaches this at + * all only because a resolver may report an already-resolved address as unresolved in order to + * redirect it (see {@link #resolveCandidates}). Its host string is not as trustworthy: it renders + * a mutable field on the shared {@link InetAddress} (see {@link AddressUtils#carriesName}), which + * holds the name the operator configured when the address was built from one -- {@code new + * InetSocketAddress("db.example.com", 9042)} resolves eagerly and keeps the name -- but holds + * whatever reverse-DNS name an earlier TLS handshake cached when it was not. The two are + * indistinguishable from the object. + * + *

Re-attaching regardless is still the better of the two, because of what the alternative + * costs. Leaving a redirected candidate unlabelled does not leave it neutral: {@code + * DefaultSslEngineFactory} derives the TLS peer host from {@code resolve()}, which for the pinned + * copy is this candidate, and for a nameless address that is a blocking reverse lookup on an + * event loop. So in the configured-name case the choice is between validating the certificate + * against the configured DNS SAN and validating it against a PTR record -- which is what the + * pre-multi-address path did, when {@code resolve()} still handed back the endpoint's own + * address. And in the cached-PTR case it is between one address's PTR name and another's, neither + * of which the operator ever wrote. One case is fixed and the other is a wash. + */ + @VisibleForTesting + static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { + if (!(original instanceof InetSocketAddress) || !(candidate instanceof InetSocketAddress)) { + return candidate; + } + InetSocketAddress originalInet = (InetSocketAddress) original; + InetSocketAddress candidateInet = (InetSocketAddress) candidate; + InetAddress candidateIp = candidateInet.getAddress(); + if (candidateIp == null) { + return candidate; + } + String hostString = originalInet.getHostString(); + if (AddressUtils.carriesName(originalInet)) { + // The queried name always wins -- unless the candidate already carries it, which is the + // common case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never + // looks anything up, and for a nameless candidate it falls back to the IP literal, which + // cannot equal a name -- so this test does not mistake one for the other. + return hostString.equals(candidateInet.getHostString()) + ? candidate + : relabel(candidateInet, candidateIp, hostString); + } + // An IP literal: re-attach it only to the address it denotes, so a redirect stays unlabelled. + // The whole original string, zone and brackets included, becomes the label; only the address + // part is matched on, which is what AddressUtils#parseLiteral hands back. + // + // Parsed there rather than here, next to the AddressUtils#isLiteral that put us on this branch + // in the first place. The two have to accept the same strings -- a literal recognised here and + // rejected by the parse returns the candidate unlabelled, which is the one outcome this branch + // exists to prevent: getHostName() would then answer with a reverse lookup and the SSL engine + // would validate against a PTR record -- and keeping the parse on this side made that agreement + // a matter of comment rather than of code. + InetAddress literal = AddressUtils.parseLiteral(hostString); + if (literal == null) { + return candidate; + } + // A byte-exact comparison, with IPv4 and IPv6 told apart by array length. This is equivalent + // to InetAddress.equals(), which ignores the scope id as well -- verified on JDK 11.0.30, where + // two Inet6Addresses built from the same bytes with scope ids 3 and 5 compare equal -- and is + // written out so that the scope-blindness is visible rather than inherited: the zone was split + // off just above and goes into the label, never into this test. + // + // The consequence is accepted, not overlooked. A candidate carrying a different scope than the + // configured zone still matches here and is relabelled with that zone, so getHostString() names + // one interface while the connect goes out on the candidate's own. Reaching that needs a + // resolver that answers a zoned literal with a different scope than it was asked about; the + // alternative -- resolving the zone name through NetworkInterface to compare it -- buys a + // NetworkInterface lookup on the connect path for that one case, so it is deferred rather than + // taken here. + if (!Arrays.equals(literal.getAddress(), candidateIp.getAddress())) { + return candidate; + } + // Deliberately no getHostString() short-circuit on this branch: a *nameless* candidate's host + // string is its own IP literal, so it compares equal to the label about to be attached and the + // relabel would be skipped -- leaving getHostName() to answer with a reverse lookup, which is + // the one thing this branch exists to prevent. Relabelling is idempotent, so paying for it + // unconditionally is cheaper than telling the two apart. + return relabel(candidateInet, candidateIp, hostString); + } + + /** + * {@code candidate} rebuilt with {@code hostName} as its label, or itself if that is not + * possible. + */ + private static SocketAddress relabel( + InetSocketAddress candidate, InetAddress candidateIp, String hostName) { + try { + return new InetSocketAddress( + AddressUtils.withHostName(hostName, candidateIp), candidate.getPort()); + } catch (UnknownHostException impossible) { + // getByAddress only rejects illegal byte lengths, and these bytes come from a real + // InetAddress; keep the raw candidate rather than failing the connect over a cosmetic step. + return candidate; + } + } + + /** + * Whether every address this endpoint expands to is another way in to the same server, which only + * the endpoint knows — see {@link PinnableEndPoint#addressesAreInterchangeable(SocketAddress)} + * for the two cases and why they differ. An endpoint that does not implement {@link + * PinnableEndPoint} is treated as not interchangeable, which is also the conservative reading for + * a third-party implementation. + * + *

Asked once per connect, in {@link #connect}, with both of the booleans that depend on + * it derived from the one answer: an endpoint backed by mutable state asked twice could answer + * about a different address than the one being dialled, and the two would then disagree. {@code + * resolvedAddress} is passed in rather than re-derived for the same reason — it is what {@link + * EndPoint#resolve()} already returned for this connect. + */ + @VisibleForTesting + static boolean addressesAreInterchangeable(EndPoint endPoint, SocketAddress resolvedAddress) { + return endPoint instanceof PinnableEndPoint + && ((PinnableEndPoint) endPoint).addressesAreInterchangeable(resolvedAddress); + } + + /** + * Whether {@link #shuffleAndLimit} may spread this connect across the addresses the endpoint + * expands to. + * + *

A contact point always may: its addresses may well be different nodes, so there is no + * node identity to preserve, and spreading both balances load and varies which address an attempt + * starts from. An {@linkplain #isIdentified(Node) identified} node may only when its addresses + * are interchangeable -- which is what {@code SniEndPoint#resolve()} used to do for itself, + * rotating through the sorted records, before resolution moved to this layer. + */ + @VisibleForTesting + static boolean spreadAcrossAddresses(boolean nodeIsIdentified, boolean interchangeable) { + return !nodeIsIdentified || interchangeable; + } + + /** + * Whether a rejection observed at one address is a verdict on the server, and so on every + * remaining address, rather than on the record that reached it. See {@link #isNodeWideFailure}, + * the only thing that asks. + * + *

An identified node always qualifies: every address of it is that same node. An unidentified + * contact point qualifies only when its addresses are interchangeable, i.e. when the endpoint + * says they all lead to one server. Otherwise they may be distinct servers running distinct + * software, which is not an edge case but what a rolling upgrade looks like from the client. + * + *

Which reads as the opposite of what {@link #shuffleAndLimit} says about the same input, and + * the difference is deliberate rather than an oversight. One {@code Node} denoting one server is + * the driver's model, and a configured {@code AddressTranslator} that hands back a name can + * violate it. The two questions answer that differently because being wrong costs differently: + * withholding the shuffle is free, so spreading hedges and keeps the resolver's order, while a + * failure has to be attributed to something and every address of an identified node is all this + * layer has. So this trusts the model, and the mirror case it leaves unrescued -- a heterogeneous + * identified node -- is owned in {@link #isNodeWideFailure}, where the cost of it is spelt out. + */ + @VisibleForTesting + static boolean sameServerAtEveryAddress(boolean nodeIsIdentified, boolean interchangeable) { + return nodeIsIdentified || interchangeable; + } + + /** + * Truncates the expanded address list to {@code advanced.connection.max-candidate-addresses}, + * shuffling it first when the addresses may be spread across (see {@link + * #spreadAcrossAddresses}). + * + *

The shuffle spreads load: without it, every connection would try the resolver's first + * address first and healthy connections would pile onto one IP, while the whole point of a + * multi-record name is usually to spread them. A fresh random order per connect also means + * successive attempts start at different addresses, with no per-name counter state to maintain + * and nothing depending on the order the resolver, or a sort, happened to choose. + * + *

Where the order is kept instead, it is because the addresses are not known to be + * interchangeable: an identified node whose endpoint is an unresolved name that may map to + * several hosts, which is what a configured {@code AddressTranslator} returns by default ({@code + * SubnetAddressTranslator} under {@code resolve-addresses = false}). Each pool connection is its + * own {@code connect()}, so shuffling there would land one {@code Node}'s channels on different + * hosts while routing, shard awareness and per-node metrics attribute them all to that node. + * Keeping the resolver's order means such a pool converges on one address, as it did before + * multi-address support -- {@code Bootstrap.connect()} resolved through {@code resolve()}, + * singular, i.e. the first record -- while the remaining addresses still serve as fallback. + * + *

The cap bounds what a single connect attempt can cost: every address tried is a full TCP + * connect plus init handshake -- and, with wrong credentials, a rejected login (see {@link + * #tryNextCandidate} on why an authentication failure does not stop the loop). For a shuffled + * list, a capped attempt tries a different sample of the addresses each time, so a name with more + * records than the cap still reaches all of them across successive attempts; one attempt just no + * longer walks them all. Where the order is kept, the cap is a hard limit -- a capped attempt + * keeps dialing the same prefix of the list, so records beyond it are never reached. That is + * accepted rather than worked around: it is still strictly more than the single address such an + * endpoint got before multi-address support, and rotating the window instead would give up the + * convergence the stable order exists for. + */ + @VisibleForTesting + List shuffleAndLimit( + List addresses, boolean spreadAcrossAddresses) { + List shuffled = new ArrayList<>(addresses); + if (shuffled.size() > 1 && spreadAcrossAddresses) { + Collections.shuffle(shuffled, random); + } + int cap = + Math.max( + 1, + context + .getConfig() + .getDefaultProfile() + .getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)); + if (shuffled.size() > cap) { + LOG.debug( + "[{}] Resolved {} addresses, will try at most {}" + + " (advanced.connection.max-candidate-addresses)", + logPrefix, + shuffled.size(), + cap); + return shuffled.subList(0, cap); + } + return shuffled; + } + + /** + * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one + * in sequence; when an address fails, the next candidate is tried, and only when all candidates + * are exhausted is the overall {@code resultFuture} failed. + * + *

Two failures are node-wide -- they doom every remaining address rather than only the + * one that was tried: an {@link UnsupportedProtocolVersionException} and an {@link + * UnsupportedEventTypeException}. Both are properties of the server rather than of the record + * that reached it, so both are gated on the same thing, {@code sameServerAtEveryAddress} (see + * {@link #sameServerAtEveryAddress}, and {@link #isNodeWideFailure} for why that is the right + * question for each). + * + *

What that covers is narrower than it looks, because {@code isNegotiating} is true only while + * {@link #protocolVersion} is still unset -- i.e. on the session's first connection, which is + * always to a contact point, and a contact point is never {@linkplain #isIdentified(Node) + * identified}. So a version rejection reached by negotiation stops the loop only when the + * contact point's endpoint reports its addresses interchangeable, as an SNI or client-routes + * proxy does. A plain multi-record name still walks the downgrade ladder from the top on every + * candidate -- each gets a fresh {@code attemptedVersions} list, so N records cost N ladders, + * with N bounded by {@link #shuffleAndLimit} -- and that is the intent, because those records may + * be different servers. + * + *

Every other failure -- authentication included -- advances to the next candidate: the + * addresses a name expands to may well belong to different nodes, so a rejection by the first of + * them says nothing about the rest. That also preserves the behaviour this PR would otherwise + * have removed: with {@code advanced.resolve-contact-points = true} each resolved address used to + * be a separate {@code Node}, and {@code ControlConnection} advances to the next node in its + * query plan on any error, including these. + * + *

Authentication in particular has to advance, for a reason only visible in the order of the + * handshake: {@link ProtocolInitHandler} runs {@code STARTUP -> AUTH_RESPONSE -> + * GET_CLUSTER_NAME}, so authentication completes before the cluster-name check. A stale + * DNS record pointing at a foreign cluster that wants different credentials therefore fails at + * AUTH, and treating that as terminal would write off the whole hostname -- making the + * cluster-name mismatch that would have advanced to the next address unreachable, in exactly the + * multi-record case this loop exists for. What bounds the cost of genuinely wrong credentials is + * the candidate cap ({@link #shuffleAndLimit}): one attempt pays at most {@code + * advanced.connection.max-candidate-addresses} rejected logins, with the earlier failures + * attached as suppressed exceptions. + * + *

Timeout note: addresses are tried serially, so the worst-case time before failure is + * N times a full attempt, and an attempt is a connect plus the init handshake. Each of the + * handshake's steps arms its own {@code advanced.connection.init-query-timeout} when it is sent, + * so they accumulate instead of sharing one deadline: a single address that accepts the + * connection and then stalls costs {@code connect-timeout} plus several times {@code + * init-query-timeout} before the loop moves on. This is an intentional tradeoff: failing + * immediately on the first unreachable IP would prevent fallback to healthy ones. The candidate + * cap ({@link #shuffleAndLimit}) is what bounds N. + * + *

When every candidate fails, one of their errors is propagated -- see {@link + * #surfacedFailure} for which, and why it is not simply the last -- with every other candidate's + * failure attached to it as a {@linkplain Throwable#addSuppressed(Throwable) suppressed} + * exception, so no cause is lost. + * + *

What that does not give is which address produced which failure. Every one of these + * exceptions is built from the endpoint, and a pinned copy is required to render identically to + * the unpinned original ({@link PinnableEndPoint}), so a three-record name yields three messages + * that all name the same hostname. The DEBUG line above is where the pairing lives; making the + * exceptions themselves carry it would mean either relaxing that contract or wrapping causes in a + * driver-owned type, which would in turn break the {@code instanceof} tests the callers do on + * them (see {@link #surfacedFailure}). + */ + private void tryNextCandidate( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + boolean sameServerAtEveryAddress, + CompletableFuture resultFuture, + List candidates, + int index, + List priorErrors) { + + // Invariant: this method always (eventually) completes resultFuture. It is invoked from + // CompletionStage and Netty callbacks that swallow throwables, so a synchronous throw -- a + // custom PinnableEndPoint.pinTo() for instance -- would otherwise leave the connect attempt + // hanging forever. Double completion is harmless: completeExceptionally() on an already + // completed future is a no-op. + try { + SocketAddress candidate = candidates.get(index); + // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and + // the DriverChannel handed to the caller -- sees an endpoint bound to this one address + // instead of the multi-address original. See PinnableEndPoint for why that matters. + EndPoint pinnedEndPoint = pin(endPoint, candidate); + CandidateFuture perAddressFuture = new CandidateFuture(); + // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one + // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms + // this candidate) only reports versions actually tried against it, not earlier candidates'. + List attemptedVersions = new CopyOnWriteArrayList<>(); + connectToAddress( + baseBootstrap, + eventLoop, + pinnedEndPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + candidate); + + perAddressFuture.whenComplete( + (channel, error) -> { + try { + boolean nodeWide = + error != null && isNodeWideFailure(error, sameServerAtEveryAddress); + if (error == null) { + if (!resultFuture.complete(channel)) { + // Same guard as completeCandidate and abandonCandidate: resultFuture is handed to + // callers as a CompletionStage and every path that can complete it early does so + // exceptionally (the blanket catches in resolveCandidates and below), so losing + // this race is possible -- and would otherwise leak a live socket and its + // pipeline for the life of the JVM, since nobody else holds this channel. + channel.forceClose(); + } + } else if (!nodeWide && index + 1 < candidates.size()) { LOG.debug( - "[{}] Failed to connect with protocol {}, retrying with {}", + "[{}] Failed to connect to {} ({}), trying next address", logPrefix, - currentVersion, - downgraded.get()); - connect( + candidate, + error.getMessage()); + priorErrors.add(error); + tryNextCandidate( + baseBootstrap, + eventLoop, + // Deliberately the original, not the pinned copy: the next candidate must be + // pinned from the unpinned endpoint. endPoint, shardingInfo, shardId, options, nodeMetricUpdater, - downgraded.get(), - true, - attemptedVersions, - resultFuture); + currentVersion, + isNegotiating, + sameServerAtEveryAddress, + resultFuture, + candidates, + index + 1, + priorErrors); + } else { + if (index + 1 < candidates.size()) { + // Only reachable for a node-wide failure (see the javadoc). + LOG.debug( + "[{}] Not trying the remaining addresses of {}: this failure is a property of" + + " the node, not of the address ({})", + logPrefix, + endPoint, + error.getMessage()); + } + // Surface one failure, carrying the others as suppressed exceptions so they are + // not lost (they were only logged at DEBUG above). Deduplicated by identity: + // nothing stops two candidates from failing with the same Throwable instance, and + // this mutates an object we do not own -- attaching it twice would show the same + // cause twice, and would keep growing a shared instance's suppressed list on every + // connect. + List allErrors = new ArrayList<>(priorErrors); + allErrors.add(error); + Throwable surfaced = surfacedFailure(allErrors, nodeWide); + Set attached = Collections.newSetFromMap(new IdentityHashMap<>()); + attached.add(surfaced); + for (Throwable candidateError : allErrors) { + if (attached.add(candidateError)) { + surfaced.addSuppressed(candidateError); + } + } + // Note: might be completed already if the failure happened in initializer() + resultFuture.completeExceptionally(surfaced); + } + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + }); + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + } + + /** + * Whether {@code error} dooms every remaining address of the endpoint, making it pointless for + * {@link #tryNextCandidate} to try them. See its javadoc for the reasoning behind each case. + * + *

A {@link ClusterNameMismatchException} is deliberately absent, for an identified node as + * much as for a contact point. It says that the address just tried fronts a different cluster, + * which is a property of that record rather than of the node -- a stale DNS entry is exactly what + * it looks like -- so advancing to the next address is the whole point. {@link #surfacedFailure} + * treats it with the same caution on the way out. + * + *

An {@link AuthenticationException} is deliberately absent too, even for an identified node: + * see {@link #tryNextCandidate} on why authentication must advance, and {@link #shuffleAndLimit} + * for the cap that bounds what wrong credentials can cost. + * + *

Both cases here are properties of the server rather than of the record that reached + * it: which protocol versions it speaks, and which event types it knows. That is why they can be + * node-wide at all -- where the same server answers everywhere, replaying either against the + * remaining addresses can only fail the same way, and each replay costs a full TCP connect plus + * the STARTUP/AUTH/cluster-name handshake and the connect hook's round trip. Stopping at the + * first restores what a rejection cost before an endpoint expanded to several addresses, which + * was one failed connect per contact point. + * + *

Where it is not the same server, that saving is given up on purpose, and the case it + * is given up for is the common one: {@link UnsupportedEventTypeException} is raised only for + * {@code CLIENT_ROUTES_CHANGE} (see {@link #translateRegisterFailure}), which only the control + * connection registers -- so on the initial connect the endpoint is always an unidentified + * contact point and this always advances. A multi-record contact point mid-rolling-upgrade is + * exactly the deployment where advancing finds the address that works, so paying one connect per + * record to find it is the trade. What must not be given up with it is the diagnosis, which is + * why {@link #surfacedFailure} gives this type a rung of its own rather than letting whichever + * address failed last speak for the endpoint. + * + *

And it is why both are gated on the same question -- whether the same server really does + * answer at every address (see {@link #connect}, which derives it). Asking only whether the + * node is identified would be wrong in both directions. Too narrow: an unidentified + * contact point behind an SNI or client-routes proxy has every address routed to one node, so a + * rejection settles all of them and replaying the downgrade ladder against each proxy IP is + * waste. Too wide: an unidentified contact point that is a plain multi-record name may front + * distinct servers, and during a rolling upgrade they genuinely differ -- writing the name off + * because the first address answered was the one not yet upgraded would skip the addresses that + * would have worked. + * + *

What that leaves unrescued is the mirror of the second case, and is accepted: a + * heterogeneous identified node, whose own addresses disagree about protocol versions or + * event types. Every address of an identified node is that node, so the driver has nowhere better + * to look. Reachable only by pointing a node at a name that covers several hosts -- see {@link + * #sameServerAtEveryAddress} -- and unchanged by this loop either way: before it, {@code + * resolve()} handed over the resolver's first record alone, so the same bad record produced the + * same verdict from a single-address connect. What the loop does not do is rescue it, and {@link + * #surfacedFailure} keeps it that way on purpose: a node-wide failure outranks the unanimity rule + * there, so one such record still forces the node down rather than being demoted to whatever + * transport error a sibling address produced. + */ + private static boolean isNodeWideFailure(Throwable error, boolean sameServerAtEveryAddress) { + return sameServerAtEveryAddress + && (error instanceof UnsupportedProtocolVersionException + || error instanceof UnsupportedEventTypeException); + } + + /** + * Which of the candidates' failures to propagate once they are all exhausted, the rest being + * attached to it as suppressed exceptions. + * + *

Not simply the last one. Callers branch on the type of what they receive -- {@link + * com.datastax.oss.driver.internal.core.pool.ChannelPool#handleError} treats a cluster-name + * mismatch and a protocol-version rejection as fatal, an invalid keyspace as a keyspace error and + * an authentication failure as warn-and-retry; {@code ControlConnection} logs authentication + * failures differently from transport ones -- and with a multi-record name the address that + * happens to be tried last is arbitrary. Letting it win would report a firewalled IP's connect + * timeout for what is really a rejected password, and take the reconnect path where the caller + * asked for the fatal one. + * + *

So a failure the callers classify is preferred over one they do not, in the order they test + * for it, and the last non-fatal failure is only used when no candidate produced a + * classified one. Every failure is still attached, so nothing is lost either way. One rung is + * there for a reader rather than a caller: an unsupported event type is nothing any caller + * branches on, but it is the only failure here that tells an operator what to change. + * + *

The two fatal types are the exception to that: they are only preferred when every + * candidate failed that way, or when the last one is the node-wide failure that stopped the loop. + * See the comments in the body. + */ + private static Throwable surfacedFailure(List errors, boolean lastIsNodeWide) { + Throwable lastError = errors.get(errors.size() - 1); + // A node-wide failure is what ended the loop, and it is a verdict about the node rather than + // about the one address it was observed on (see tryNextCandidate). It therefore outranks + // everything below, including the unanimity rule -- which would otherwise demote it to whatever + // transport failure an earlier address happened to produce, turning the forced-down node that a + // single-address connect has always produced into a reconnect. + if (lastIsNodeWide) { + return lastError; + } + // An irreversible verdict needs evidence from every address. handleError turns these two into + // TopologyEvent.forceDown, and nothing in the driver ever reverses one -- no component fires + // FORCE_UP, and a SUGGEST_UP is explicitly refused for a FORCED_DOWN node -- so the node is out + // for the rest of the session. Meanwhile tryNextCandidate() classifies a cluster-name mismatch + // as a property of the address and advances past it, which is the point: it means this record + // is stale, not that this node belongs to another cluster. Promoting one such record over the + // other candidates' transport failures would write a healthy node off on the strength of the + // one address that was never going to work. Requiring unanimity leaves a single-address + // endpoint exactly as it was before this loop existed -- one candidate is unanimous by + // definition -- and a mixed pass simply reconnects, forcing down on the first pass that is. + boolean everyCandidateFatal = true; + for (Throwable error : errors) { + if (!isFatalToCallers(error)) { + everyCandidateFatal = false; + break; + } + } + if (everyCandidateFatal) { + return errors.get(0); + } + // An invalid keyspace, unlike those two, is a property of the cluster's schema rather than of + // the address, so one address answering settles it and no unanimity is required. It outranks an + // authentication failure because it is the rung a caller acts on -- handleError routes it to + // onKeyspaceError, which is how PoolManager fails session init fast instead of reconnecting for + // a keyspace that will never appear -- and because reaching the keyspace step at all proves the + // credentials were accepted on that address. + for (Throwable error : errors) { + if (error instanceof InvalidKeyspaceException) { + return error; + } + } + for (Throwable error : errors) { + if (error instanceof AuthenticationException) { + return error; + } + } + // An event type the server does not know is a verdict about the deployment, and the message + // says what to do about it ("requires ScyllaDB Enterprise >= ..."), so it outranks an + // unclassified transport failure that happened to come later. Without a rung of its own it + // would fall through to the last non-fatal failure below and ClientRoutesTopologyMonitor#init + // would report a bare connect timeout, with the actionable message reachable only through + // getSuppressed(). + // + // Below the authentication rung, not above it: if the credentials were rejected, whether the + // server also speaks CLIENT_ROUTES_CHANGE was never established. And no unanimity is required, + // for the same reason as the invalid keyspace above -- one server answering settles what the + // software supports. It cannot force a node down (see #isFatalToCallers), so promoting it + // costs nothing irreversible. + for (Throwable error : errors) { + if (error instanceof UnsupportedEventTypeException) { + return error; + } + } + // The last failure -- but not a fatal one. The address tried last is arbitrary, so promoting a + // fatal failure here would force the node down on the strength of the single address that + // produced it, which is exactly what the unanimity rule above refuses to do. + for (int i = errors.size() - 1; i >= 0; i--) { + Throwable error = errors.get(i); + if (!isFatalToCallers(error)) { + return error; + } + } + // Unreachable: a list of nothing but fatal failures returned at the unanimity check above. + return lastError; + } + + /** + * Whether the callers treat {@code error} as fatal, i.e. as grounds to write the node off: {@code + * ChannelPool#handleError} turns these two, and only these two, into {@code + * TopologyEvent.forceDown}. + */ + private static boolean isFatalToCallers(Throwable error) { + return error instanceof ClusterNameMismatchException + || error instanceof UnsupportedProtocolVersionException; + } + + /** + * Whether {@code error} and every failure attached to it are authentication failures, i.e. + * whether "authentication" is the whole story for the endpoint that produced it. + * + *

The test callers must use in place of a bare {@code instanceof AuthenticationException}, and + * it lives here because this class is what makes the bare test wrong: one failure no longer means + * one address. A connect expands an endpoint to every address it resolves to and reports a single + * failure for the endpoint, with the others attached as {@linkplain Throwable#getSuppressed() + * suppressed} exceptions -- and {@link #surfacedFailure} deliberately promotes an authentication + * failure over transport ones, so an endpoint whose records failed {@code [refused, refused, + * auth]} surfaces the auth error. Counting that as {@code errors.connection.auth} alone, and + * telling the operator their credentials are wrong, hides that two thirds of the deployment is + * unreachable. + * + * @see com.datastax.oss.driver.internal.core.pool.ChannelPool + * @see com.datastax.oss.driver.internal.core.control.ControlConnection + */ + public static boolean isAuthOnly(Throwable error) { + return isAuthOnly(error, ignored -> false); + } + + /** + * {@link #isAuthOnly(Throwable)}, with some of the attached failures set aside as no evidence + * either way. + * + *

For the caller that has a class of failure which says nothing about credentials, and must + * not let it decide the question. {@code ControlConnection} passes its own exclusions: a node the + * connection was not allowed to use was never asked for a password, so a contact point whose + * addresses went {@code [excluded, auth]} is an authentication failure and nothing else. Testing + * it with the plain method would find the exclusion among the suppressed and answer {@code + * false}, which is how one such contact point comes to veto the verdict for a whole round. + * + *

Setting everything aside is not an authentication failure. At least one real one has to + * remain, or an endpoint that was only ever refused would report as an auth failure -- the + * opposite mistake, and the one the caller's own skip is there to prevent. + */ + public static boolean isAuthOnly(Throwable error, Predicate ignore) { + boolean anyAuth = false; + if (error instanceof AuthenticationException) { + anyAuth = true; + } else if (!ignore.test(error)) { + return false; + } + for (Throwable suppressed : error.getSuppressed()) { + if (suppressed instanceof AuthenticationException) { + anyAuth = true; + } else if (!ignore.test(suppressed)) { + return false; + } + } + return anyAuth; + } + + /** + * Whether an authentication failure appears anywhere in what {@code error} reports -- as the + * failure itself or as one of the {@linkplain Throwable#getSuppressed() suppressed} ones. + * + *

The counterpart to {@link #isAuthOnly}, for the caller that needs to know a login was + * rejected at all rather than whether that is the whole story. {@link #surfacedFailure} promotes + * an invalid keyspace, and a node-wide failure, over an authentication failure, so for an + * endpoint that expands to several addresses the auth error is routinely not the one that comes + * out -- and a caller testing the type of what it received would never see it. + */ + public static boolean mentionsAuthentication(Throwable error) { + if (error instanceof AuthenticationException) { + return true; + } + for (Throwable suppressed : error.getSuppressed()) { + if (suppressed instanceof AuthenticationException) { + return true; + } + } + return false; + } + + /** + * Performs a Netty bootstrap connect to a single, already-resolved address. Handles + * protocol-version negotiation (downgrade retries) internally, staying on the same address. Uses + * {@code perAddressFuture} so {@link #tryNextCandidate} can distinguish a per-address TCP failure + * (try the next IP) from a successful protocol handshake. + */ + private void connectToAddress( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CandidateFuture perAddressFuture, + SocketAddress resolvedAddress) { + + if (shardId == null || shardingInfo == null) { + if (shardId != null) { + LOG.debug( + "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", + shardId, + endPoint); + } + bootstrapAndConnect( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + resolvedAddress, + null); + return; + } + + // Picking a shard-aware local port means probing ports with ServerSocket.bind(): a blocking + // syscall per probe, and when the range is contended a scan across the whole of + // [port-low, port-high] -- twice, if the first pass wraps (see PortAllocator). + // + // That must not run on `eventLoop`. It is one of the I/O loops, shared with every established + // channel registered on it, and advanced shard awareness is enabled by default, so this is the + // ordinary path against Scylla rather than an edge case. The loop is reached by two separate + // routes -- resolveCandidates() completes its future there, and the downgrade retry below + // re-enters this method from inside a Netty listener -- so the guard belongs here, at the + // blocking call, rather than at either caller. + // + // The admin group, because that is where this scan already ran: before resolution moved into + // this class, connect() did it inline on its calling thread, which is the adminExecutor of the + // ChannelPool or ControlConnection driving the connect. It carries no request traffic. + // + // Two things do differ from that, both accepted rather than unnoticed. next() takes a thread + // from the group (advanced.netty.admin-group.size, 2 by default) instead of the caller's own, + // so a pool's scan can now land on the thread the control connection runs on, where before it + // could only stall the caller's own queue. And the candidate loop reaches this once per address + // tried rather than once per connect(), so an endpoint whose earlier addresses fail pays it + // more than once. Both stay bounded -- the loop is sequential within a connect, and + // max-candidate-addresses caps the addresses at 5 -- and neither puts blocking work on this + // group that was not already there. Moving the scan off the control plane altogether is the + // real fix and is tracked in the deferred ledger. + try { + context + .getNettyOptions() + .adminEventExecutorGroup() + .next() + .execute( + () -> { + // The same invariant as below, restated because it is a fresh entry point: this + // body runs as an executor task, and Netty only logs a task's throwables. + try { + int localPort = + PortAllocator.getNextAvailablePort( + shardingInfo.getShardsCount(), shardId, context); + if (localPort == -1) { + LOG.warn( + "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", + shardId, + endPoint); + } + bootstrapAndConnect( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + resolvedAddress, + localPort == -1 ? null : localPort); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); + } + }); + } catch (Throwable t) { + // RejectedExecutionException, if the group is shutting down. Completing the future here is + // what keeps a connect from hanging on it. + perAddressFuture.completeExceptionally(t); + } + } + + /** + * The rest of {@link #connectToAddress}, once the local port (if any) has been settled. + * + * @param localPort the local port to bind to, or {@code null} to let the OS pick one. + */ + private void bootstrapAndConnect( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CandidateFuture perAddressFuture, + SocketAddress resolvedAddress, + Integer localPort) { + + // Invariant, as in tryNextCandidate(): every path completes perAddressFuture. The synchronous + // section can throw from Bootstrap validation; the connect listener runs inside a Netty + // callback that swallows throwables and contains the downgrade recursion, the version-registry + // lookup and the config overrides, any of which throwing would otherwise hang the attempt. + try { + // Captured here, beside the pipeline that is about to be built, because ProtocolInitHandler + // snapshots this same option in its constructor (its `timeoutMillis`) and every init step + // then runs on that one value. Reading it again later -- REGISTER is sent after init now, so + // it would be a second read -- lets a config reload landing inside this connect apply a new + // value to a connection that already exists, which is precisely the scope reference.conf + // rules out: "the new value will be used for connections created after the change". This is + // not quite the same instant as ProtocolInitHandler's own read -- that one happens in + // initChannel, once the connect below registers the channel -- so a reload landing in + // between would still split the two. What closes is the window that matters: the one + // spanning the whole handshake, from STARTUP to the REGISTER that now follows it. + // + // It also removes a way to hang the attempt. The two request classes disagree about a + // non-positive value -- AdminRequestHandler#onWriteComplete arms no timer at all, while the + // ChannelHandlerRequest the init steps use arms one unconditionally -- so a reload to zero + // mid-handshake used to leave STARTUP bounded by the old value and REGISTER bounded by + // nothing, on a connection with no hook backstop behind it. + Duration initQueryTimeout = + context + .getConfig() + .getDefaultProfile() + .getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT); + + // clone(eventLoop) so each attempt gets its own handler while sharing the options (including + // anything afterBootstrapInitialized() set), and is registered on the event loop the + // connect() picked -- the same one resolution ran on, so the group's chooser advances exactly + // once per logical connect (see connect()). + // + // disableResolver() because resolveCandidates() has already done the one resolution pass this + // connect gets, and `resolvedAddress` is one of its results. Bootstrap.clone() otherwise + // carries the resolver over and Netty resolves again -- through resolve(), *singular*. That + // is inert for the default resolver, which short-circuits on isResolved(), but a resolver + // that reports resolved addresses as unresolved in order to redirect them -- which + // resolveCandidates() deliberately supports -- would remap every candidate onto its first + // answer: the remaining candidates would never actually be tried, and the endpoint pinned + // onto the channel would name an address the channel is not connected to (which is what the + // SSL engine's peer host and DefaultTopologyMonitor#savePort are derived from). Every other + // exit from resolveCandidates() yields an address Netty would itself have passed through + // untouched -- no group, !isSupported, or isResolved -- so nothing else changes. + Bootstrap bootstrap = + baseBootstrap + .clone(eventLoop) + .disableResolver() + .handler( + initializer( + endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); + + ChannelFuture connectFuture = + (localPort == null) + ? bootstrap.connect(resolvedAddress) + : bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + + connectFuture.addListener( + cf -> { + try { + if (connectFuture.isSuccess()) { + Channel channel = connectFuture.channel(); + DriverChannel driverChannel = + new DriverChannel( + endPoint, channel, context.getWriteCoalescer(), currentVersion); + finishCandidate( + driverChannel, + options, + initQueryTimeout, + perAddressFuture, + () -> latchNegotiatedState(driverChannel, currentVersion, isNegotiating)); } else { - resultFuture.completeExceptionally( - UnsupportedProtocolVersionException.forNegotiation( - endPoint, attemptedVersions)); + Throwable error = connectFuture.cause(); + if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { + attemptedVersions.add(currentVersion); + Optional downgraded = + context.getProtocolVersionRegistry().downgrade(currentVersion); + if (downgraded.isPresent()) { + LOG.debug( + "[{}] Failed to connect with protocol {}, retrying with {}", + logPrefix, + currentVersion, + downgraded.get()); + // Stay on the same address for protocol-version downgrade retries. + connectToAddress( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + downgraded.get(), + true, + attemptedVersions, + perAddressFuture, + resolvedAddress); + } else { + perAddressFuture.completeExceptionally( + UnsupportedProtocolVersionException.forNegotiation( + endPoint, attemptedVersions)); + } + } else { + // Note: might be completed already if the failure happened in initializer(), this + // is fine + perAddressFuture.completeExceptionally(error); + } + } + } catch (Throwable t) { + // Close the channel we opened before giving up on it. Nothing else holds it once this + // listener returns -- the DriverChannel wrapper is out of scope, and no candidate was + // completed with it -- so the socket and its pipeline would stay open for the life of + // the JVM. Only when the future is ours to fail, though: a candidate that completed + // successfully is the caller's channel, not ours to close. + if ((perAddressFuture.completeExceptionally(t) + || perAddressFuture.isCompletedExceptionally()) + && connectFuture.isSuccess()) { + connectFuture.channel().close(); } + } + }); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); + } + } + + /** + * Remembers what the first accepted connection negotiated, so later connections skip the + * negotiation: the protocol version, the cluster name to check others against, and the server's + * product type (which for Cloud also lowers the default consistency level). + * + *

Run only once a candidate has been accepted, not as soon as its transport connect and + * init handshake succeed. Init is no longer the last word on a candidate: the connect hook can + * reject it (the control connection's identity read does, for a node with no {@code host_id}), + * and REGISTER, which used to be the final init step, now runs after that hook. A candidate the + * driver is about to throw away must not leave its cluster name latched here -- a stale DNS + * record pointing at a foreign cluster would otherwise make every subsequent connection fail its + * cluster-name check, which {@code ChannelPool} turns into an irreversible forced-down node. + * + *

What enforces that is {@link CandidateFuture#settle()}: only the candidate that wins it + * reaches {@link #completeCandidate}, and only {@link #completeCandidate} runs this. Note that + * "accepted" is per connect attempt, not per factory -- two concurrent negotiating connects each + * accept a candidate and each latch, so {@code protocolVersion} can legitimately be written more + * than once (it has no first-write-wins guard, unlike the two below). Both writers negotiated + * against the same cluster, so they agree except while it is being upgraded. + */ + private void latchNegotiatedState( + DriverChannel driverChannel, ProtocolVersion currentVersion, boolean isNegotiating) { + if (isNegotiating) { + this.protocolVersion = currentVersion; + } + if (this.clusterName == null) { + this.clusterName = driverChannel.getClusterName(); + } + Map> supportedOptions = driverChannel.getOptions(); + if (this.productType == null && supportedOptions != null) { + List productTypes = supportedOptions.get("PRODUCT_TYPE"); + String productType = + productTypes != null && !productTypes.isEmpty() + ? productTypes.get(0) + : UNKNOWN_PRODUCT_TYPE; + this.productType = productType; + DriverConfig driverConfig = context.getConfig(); + if (driverConfig instanceof TypesafeDriverConfig + && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { + ((TypesafeDriverConfig) driverConfig) + .overrideDefaults( + ImmutableMap.of( + DefaultDriverOption.REQUEST_CONSISTENCY, ConsistencyLevel.LOCAL_QUORUM.name())); + } + } + } + + /** + * Disarms the connect-hook backstop, if one was armed. + * + *

{@code null} when the timeout was disabled, which is why the null check lives here rather + * than at each of the three call sites. + * + *

The return value is deliberately ignored. Netty's {@code Timeout#cancel()} answers false + * both for a task already cancelled and for one currently expiring, so it distinguishes nothing a + * caller here could act on: the timer task and this thread both funnel into {@link + * #abandonCandidate}, whose {@code settle()} latch decides which of them owns the failure. + * Cancelling is an optimization -- it keeps a wheel slot from holding a dead reference for the + * rest of the timeout -- not a synchronization point. + */ + private static void cancelQuietly(Timeout hookTimeout) { + if (hookTimeout != null) { + hookTimeout.cancel(); + } + } + + /** + * The tail of a candidate attempt, once transport connect and protocol initialization have both + * succeeded: runs the caller's {@link ConnectHook} (if any), then registers for protocol events + * (if requested), and only then completes the candidate's future. + * + *

Both steps happen while this attempt still holds the endpoint's remaining addresses, so a + * failure in either is settled inside the loop: the channel is force-closed on the spot and + * {@link #tryNextCandidate} decides what to do with the addresses that are left. Usually that + * means advancing to the next address; the exception is a failure {@link #isNodeWideFailure} + * classifies, which stops the loop because it describes the server rather than the address it was + * seen on. REGISTER can produce one -- see {@link #registerForEvents}. + * + *

REGISTER used to be the last protocol-init step; it moved behind the hook so that a channel + * the hook is about to reject never registers for events. The window in which a live channel is + * not yet registered grows by the hook's round trip -- the same order of cost as the init step it + * follows. + */ + private void finishCandidate( + DriverChannel driverChannel, + DriverChannelOptions options, + Duration initQueryTimeout, + CandidateFuture perAddressFuture, + Runnable onAccepted) { + if (options.connectHook == null) { + registerForEvents(driverChannel, options, initQueryTimeout, perAddressFuture, onAccepted); + return; + } + // The hook's contract says its stage eventually completes, but only the driver can make that + // true: a wedged hook (a topology monitor whose own timeout is broken, say) would otherwise + // hang the whole connect attempt, and with it control-connection init or a reconnect. + // + // Armed before the hook is called, not after it returns. A hook that blocks inside onConnect + // never returns, so arming afterwards would not bound it on any thread -- the arming statement + // is simply never reached, which is a different failure from the one the thread choice below + // addresses and is not fixed by it. The cost of arming first is a + // timeout scheduled and cancelled again for every candidate whose hook answers promptly, which + // is a wheel insertion and a flag. + // + // What the timer can and cannot do for a blocking hook is worth being exact about: it releases + // the *connect*, not the loop. abandonCandidate completes perAddressFuture from the timer + // thread, so the Reconnection stops waiting and a later attempt can be made; the hook goes on + // holding the channel's event loop until it returns, and every other channel registered there + // stays stalled meanwhile. Bounding the connect is what is on offer, and it is worth having. + // + // Unless the timeout is zero or negative, which every other consumer of a driver timeout option + // reads as "no timeout" (see AdminRequestHandler#onWriteComplete). Scheduling it anyway would + // fire on the next event-loop turn, before any round trip can complete, and abandon every + // candidate of every contact point -- so an operator who disabled the control-connection + // timeout + // would find that the session cannot initialize at all. + // + // On the driver's timer, and neither of the two threads this connect is already using. + // + // Not this channel's event loop. The hook runs there -- this method is reached from a + // channel-promise listener, which Netty notifies on it -- so a hook that wedges that loop would + // keep it from ever dequeuing a task armed on it. Blocking is exactly what + // TopologyMonitor#getChannelNodeInfo's contract has to ask implementations not to do, because + // nothing enforces it, and it takes two shapes that need different answers: a hook that returns + // a stage and separately stalls the loop is caught by arming off that loop, and a hook that + // blocks inside onConnect is caught only by arming before the call. Neither is caught by the + // hook's own machinery. A hook that simply never completes its stage is caught wherever the + // timer lives. + // + // Not the admin group either, for a weaker version of the same reason: #connectToAddress + // dispatches the shard-aware port scan there, and that scan blocks -- a bind() probe per port + // across advanced.shard-awareness.port-{low,high} -- once per candidate address. The group is + // two threads by default and one of them is the control connection's own executor, so a + // backstop armed on it can be sitting behind the very kind of work it exists to bound. Delay + // rather than deadlock, but there is no reason to accept even that. + // + // The timer is a thread of its own, and what it carries is only ever timeout callbacks -- it is + // already where every request deadline in the driver lives (CqlRequestHandler, + // CqlPrepareHandler, the graph and continuous-paging handlers, metrics expiry), so one task per + // candidate is nothing beside its existing traffic, and none of that traffic blocks. And + // abandonCandidate needs nothing from any particular thread: the settle() latch is an + // AtomicBoolean and forceClose() is safe from any of them. Its granularity is + // advanced.netty.timer.tick-duration, 100ms by default, against a timeout measured in seconds. + Timeout hookTimeout; + try { + hookTimeout = + (options.connectHookTimeout == null || options.connectHookTimeout.toNanos() <= 0) + ? null + : context + .getNettyOptions() + .getTimer() + .newTimeout( + timeout -> + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Connect hook timed out after " + options.connectHookTimeout, + null)), + options.connectHookTimeout.toNanos(), + TimeUnit.NANOSECONDS); + } catch (Throwable t) { + // A stopped timer rejects the task -- NettyOptions#onClose, or a custom implementation that + // caps pending timeouts, which the driver's own does not. Fail the candidate rather than run + // the hook with nothing bounding it: this method is called from a Netty listener that + // swallows throwables, so the attempt would otherwise hang (see connectToAddress's + // invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Could not schedule the connect hook timeout", t)); + return; + } + CompletionStage vetted; + try { + vetted = options.connectHook.onConnect(driverChannel); + } catch (Throwable t) { + // A synchronous throw is a rejection, like an exceptional stage. Blanket-caught: this runs + // inside a Netty listener that swallows throwables, so a caller-supplied callback leaking + // one would otherwise leave the attempt hanging forever. + cancelQuietly(hookTimeout); + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", t)); + return; + } + if (vetted == null) { + cancelQuietly(hookTimeout); + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook returned a null stage", null)); + return; + } + vetted.whenComplete( + (aVoid, error) -> { + try { + cancelQuietly(hookTimeout); + if (error != null) { + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", error)); } else { - // Note: might be completed already if the failure happened in initializer(), this is - // fine - resultFuture.completeExceptionally(error); + registerForEvents( + driverChannel, options, initQueryTimeout, perAddressFuture, onAccepted); } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and the timeout that would have failed the candidate has just been + // cancelled, so anything escaping here -- registerForEvents' config read, for instance + // -- would leave perAddressFuture uncompleted forever. + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Unexpected error after the connect hook accepted the channel", t)); } }); } + /** + * Sends the REGISTER request when the options ask for protocol events, then completes the + * candidate. + * + *

A registration failure is normally a per-candidate failure, and the attempt moves to the + * next address. One is not: {@link #translateRegisterFailure} mints an {@link + * UnsupportedEventTypeException} for a server that rejects the event type outright, and {@link + * #isNodeWideFailure} treats that as node-wide wherever the same server answers at every address + * -- which is every SNI or client-routes contact point, i.e. exactly the deployments that ask for + * {@code CLIENT_ROUTES_CHANGE} in the first place. Replaying the rejection against each proxy IP + * would only collect the same answer. That is a deliberate difference from the days when REGISTER + * was a protocol-init step, where the failure took the whole endpoint down and there were no + * other addresses to take with it. + */ + private void registerForEvents( + DriverChannel driverChannel, + DriverChannelOptions options, + Duration initQueryTimeout, + CandidateFuture perAddressFuture, + Runnable onAccepted) { + if (options.eventTypes.isEmpty()) { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + return; + } + // initQueryTimeout is the value bootstrapAndConnect captured for this attempt, not a fresh + // read: this request runs after protocol initialization, so reading the option again here would + // give a connection that already exists a value configured after it was created. See the + // capture site for why that is both a documented-scope violation and, at zero, a way to leave + // this request unbounded. + // The owner's prefix plus the channel id, matching what ProtocolInitHandler builds for the + // steps that used to send this request. This factory's own logPrefix is the session name, + // which for a REGISTER timeout would name neither the connection pool nor the channel that + // timed out -- and with an endpoint expanding to several addresses there can be more than one + // of these in flight for the same node. + // Same derivation as ProtocolInitHandler#channelActive: DriverChannel#toString delegates to + // the Netty channel, whose toString is "[id: 0x..., L:... - R:...]", and the brackets come off. + String channelId = driverChannel.toString(); + channelId = channelId.length() > 1 ? channelId.substring(1, channelId.length() - 1) : channelId; + AdminRequestHandler.register( + driverChannel, + options.eventTypes, + initQueryTimeout, + options.ownerLogPrefix + "|" + channelId) + .start() + .whenComplete( + (aVoid, error) -> { + try { + if (error != null) { + abandonCandidate( + driverChannel, perAddressFuture, translateRegisterFailure(error)); + } else { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and by this point no timeout is left to fail the candidate, so + // anything escaping -- translateRegisterFailure's casts, a forceClose() on an event + // loop that is shutting down -- would leave perAddressFuture uncompleted forever + // and hang the connect attempt (see connectToAddress's invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Unexpected error after REGISTER", t)); + } + }); + } + + /** + * Gives the one REGISTER rejection with a known cause a message that names it: the server not + * knowing the {@code CLIENT_ROUTES_CHANGE} event type. This translation lived in the init handler + * when REGISTER was an init step, and exists so that the caller + * (ClientRoutesTopologyMonitor.init()) reports a clear error instead of silently degrading. + */ + private static Throwable translateRegisterFailure(Throwable error) { + if (error instanceof UnexpectedResponseException) { + Message response = ((UnexpectedResponseException) error).message; + if (response instanceof com.datastax.oss.protocol.internal.response.Error) { + com.datastax.oss.protocol.internal.response.Error protocolError = + (com.datastax.oss.protocol.internal.response.Error) response; + if (protocolError.code == ProtocolConstants.ErrorCode.PROTOCOL_ERROR + && protocolError.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { + return new UnsupportedEventTypeException( + "Server does not support CLIENT_ROUTES_CHANGE event " + + "(requires ScyllaDB Enterprise >= 2026.1). " + + "Either upgrade the server or remove the client routes configuration.", + error); + } + // Any other server error naming REGISTER. Reported the way ProtocolInitHandler reported it + // while REGISTER was an init step -- error code name included, which + // UnexpectedResponseException does not carry (its message renders the Error as + // "ERROR()", dropping the code). The type is deliberately not what the init handler + // produced: that path ended in failOnUnexpected(), whose IllegalArgumentException is + // neither a DriverException nor especially informative about what failed. + return new ConnectionInitException( + String.format( + "REGISTER: server replied with unexpected error code [%s]: %s", + ProtocolUtils.errorCodeString(protocolError.code), protocolError.message), + error); + } + } + return error; + } + + /** + * A REGISTER rejection that is a property of the server -- it does not know an event type + * the driver asked for -- rather than of the address that was dialled. + * + *

A {@link ConnectionInitException}, so that callers which branch on the type (see {@link + * #surfacedFailure}, and {@code ClientRoutesTopologyMonitor#init}, which reports the message) + * treat it exactly as they treated the same rejection when REGISTER was an init step. The subtype + * exists only so {@link #isNodeWideFailure} can recognise it. + */ + @VisibleForTesting + static class UnsupportedEventTypeException extends ConnectionInitException { + UnsupportedEventTypeException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * One candidate address's future, together with the one-shot latch that says which of {@link + * #completeCandidate} and {@link #abandonCandidate} owns the outcome. + * + *

A separate latch rather than the future's own completion state, because the two decisions + * have to be made in opposite orders. A candidate must be known accepted before it + * publishes, so that {@link #latchNegotiatedState} has already run by the time any caller can + * hold the channel -- while {@code complete()} only reports whether it won after + * publishing. Settling first separates the two: the winner latches and then publishes, the loser + * touches neither. + */ + private static class CandidateFuture extends CompletableFuture { + + // newIncompleteFuture() is deliberately not overridden: a derived stage carrying its own copy + // of the latch would imply a guarantee it does not have, and nothing needs one here -- the only + // stage derived from this future is the discarded return of tryNextCandidate's whenComplete. + + private final AtomicBoolean settled = new AtomicBoolean(); + + /** Whether the caller is the one that gets to decide this candidate's outcome. */ + boolean settle() { + return settled.compareAndSet(false, true); + } + } + + /** + * Records what the candidate negotiated and publishes its channel, in that order. + * + *

{@code onAccepted} -- see {@link #latchNegotiatedState} -- runs before the channel is + * published. {@code complete()} drives the downstream continuations synchronously, so the moment + * it returns a caller on another thread may already hold the channel; latching afterwards leaves + * a window in which it does while {@link #getProtocolVersion()} still sees {@code null} and + * throws its "not known yet" precondition, and in which a concurrently-built channel reads a null + * {@code clusterName} and skips the cluster-name check. The fields are {@code volatile}, so that + * is ordering rather than visibility -- but the window is real, and it is what made {@code + * ChannelFactoryProtocolNegotiationTest} await the value instead of reading it. + * + *

Latching first is only safe because {@link CandidateFuture#settle()} has already decided the + * outcome. Latching unconditionally would not be: a candidate the hook timeout has abandoned + * would still leave its cluster name behind, which is exactly what {@link #latchNegotiatedState} + * must not allow. + * + *

Losing the latch means the channel is nobody's -- the winner failed the future and will not + * be handed this channel -- so it is closed here rather than leaked. + * + *

Winning it carries the opposite duty: this call must then complete the future on every + * path, including a throwing {@code onAccepted}. Every blanket catch downstream of {@link + * #finishCandidate} discharges the "always completes {@code perAddressFuture}" invariant by + * calling {@link #abandonCandidate}, and that is a no-op once the candidate is settled -- so a + * throw escaping here would leave the future settled but never completed, hanging the connect + * with {@code Reconnection} stuck in ATTEMPT_IN_PROGRESS and leaking the channel, with no timeout + * left to rescue it (REGISTER has completed and the hook timeout is already cancelled). {@link + * #latchNegotiatedState} is not throw-free: on the Cloud path it reaches {@code + * TypesafeDriverConfig#overrideDefaults}, which re-parses the whole configuration. + */ + private static void completeCandidate( + DriverChannel driverChannel, CandidateFuture perAddressFuture, Runnable onAccepted) { + if (!perAddressFuture.settle()) { + driverChannel.forceClose(); + return; + } + try { + onAccepted.run(); + } catch (Throwable t) { + // Settling made this the only call that can still complete the future -- see the javadoc. + perAddressFuture.completeExceptionally(t); + driverChannel.forceClose(); + return; + } + if (!perAddressFuture.complete(driverChannel)) { + // Defensive. Several paths complete the future without settling it: the blanket catches in + // connectToAddress and bootstrapAndConnect, and ChannelFactoryInitializer#initChannel. One + // of them -- bootstrapAndConnect's connect-listener catch -- wraps the whole listener body + // and so can fire with this very channel already built. None can reach here today, all being + // upstream of the hook and REGISTER, but an unpublished channel nobody holds is a leak. + driverChannel.forceClose(); + } + } + + /** + * Closes a candidate channel that will not be used and fails its future -- unless that channel + * has meanwhile been handed to the caller, in which case it is theirs and must be left alone. + */ + private static void abandonCandidate( + DriverChannel driverChannel, CandidateFuture perAddressFuture, Throwable error) { + // The hook timeout and the hook's own completion race, and the hook's stage may complete off + // the channel's event loop -- the contract allows it, and a custom TopologyMonitor behind the + // control connection's hook is free to -- so cancel(false) can lose to a timeout task that has + // already started running. Losing the settle means completeCandidate got there first, so the + // channel is the caller's: closing it would leave them owning a dead channel with no error to + // explain it, and this error is moot anyway. + // + // Winning it makes the channel ours even if the future was already failed elsewhere (a blanket + // catch in the connect listener), in which case completeExceptionally is a no-op and the close + // is the point. forceClose is idempotent. + if (!perAddressFuture.settle()) { + return; + } + perAddressFuture.completeExceptionally(error); + driverChannel.forceClose(); + } + + /** + * Binds {@code endPoint} to the address a connection is being opened to, when the implementation + * supports it. + * + *

Third-party {@link EndPoint}s that do not implement {@link PinnableEndPoint} are returned + * unchanged, so they keep behaving exactly as they did before multi-address support: the channel + * carries the endpoint it was given. + * + *

So is an endpoint whose candidate came back unresolved. {@link + * PinnableEndPoint#pinTo(SocketAddress)} is documented to take an address that is already + * resolved, and one path through {@link #resolveCandidates} does not provide one: a resolver that + * reports the address already resolved is taken at its word and the name goes out untouched (the + * other two pass-throughs materialize an IP literal or fail, and {@link #dropUnresolved} removes + * unresolved results of {@code resolveAll}). For an endpoint that hands out a hostname, pinning + * there would freeze it on a name that still re-expands on every connect: no address stability + * gained, and whatever the endpoint does instead of consulting its own source once pinned is + * lost. + */ + private static EndPoint pin(EndPoint endPoint, SocketAddress resolvedAddress) { + if (resolvedAddress instanceof InetSocketAddress + && ((InetSocketAddress) resolvedAddress).isUnresolved()) { + return endPoint; + } + return endPoint instanceof PinnableEndPoint + ? ((PinnableEndPoint) endPoint).pinTo(resolvedAddress) + : endPoint; + } + @VisibleForTesting ChannelInitializer initializer( EndPoint endPoint, @@ -463,7 +2339,11 @@ protected void initChannel(Channel channel) { context.getNettyOptions().afterChannelInitialized(channel); } catch (Throwable t) { // If the init handler throws an exception, Netty swallows it and closes the channel. We - // want to propagate it instead, so fail the outer future (the result of connect()). + // want to propagate it instead, so fail this candidate's future. Note that is the + // per-address one, not the result of connect(): a pipeline failure that is not specific to + // the address (a bad truststore, say) therefore advances to the next candidate and is + // retried against each of them, which tryNextCandidate() documents as the deliberate + // trade-off for not being able to tell the two apart. resultFuture.completeExceptionally(t); throw t; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java new file mode 100644 index 00000000000..1b41a521a3d --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.concurrent.CompletionStage; + +/** + * A caller-supplied step that runs against every candidate channel a {@link ChannelFactory#connect} + * attempt opens, after protocol initialization succeeds and before the attempt is considered + * successful. + * + *

Its position is what makes it useful: a connect attempt may try several addresses when the + * endpoint's name resolves to more than one, and the hook runs while the factory still holds the + * remaining ones. Completing the returned stage exceptionally (or throwing synchronously) rejects + * the candidate -- the factory closes the channel and moves on to the endpoint's next address -- so + * a caller can impose its own acceptance criteria on a channel, per address, without losing the + * fallback. The control connection uses this to read {@code system.local} and refuse a channel + * whose node cannot identify itself, channeling what it read straight into its own state (see + * {@code ControlConnection}). + * + *

Contract: + * + *

+ * + *

When the options also request protocol events ({@link DriverChannelOptions#eventTypes}), the + * {@code REGISTER} request is sent after the hook completes successfully, so a channel that is + * about to be rejected never registers for events. + */ +public interface ConnectHook { + + /** + * Vets a candidate channel that completed protocol initialization. + * + * @return a stage that completes normally to accept the channel, or exceptionally to reject it + * and make the connect attempt move on to the endpoint's next address. + */ + @NonNull + CompletionStage onConnect(@NonNull DriverChannel channel); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java index 378fd2dc0b8..7f8d768153d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; +import java.time.Duration; import java.util.Collections; import java.util.List; import net.jcip.annotations.Immutable; @@ -54,17 +55,41 @@ public static Builder builder() { */ public final boolean reportConfig; + /** + * A step the caller runs against every candidate channel after protocol initialization, with the + * power to reject it while {@link ChannelFactory} still holds the endpoint's other addresses, or + * {@code null} if the caller has no vetting to do. Precedent for a behavioral member here: {@link + * #eventCallback}. + * + *

The control connection supplies one when connecting to a node whose {@code host_id} is not + * yet known -- a contact point, the one case with something to learn -- to read {@code + * system.local} and refuse a channel whose node cannot identify itself. + * + * @see ConnectHook + */ + public final ConnectHook connectHook; + + /** + * How long the factory waits for {@link #connectHook}'s stage before treating the candidate as + * rejected. Never null when {@link #connectHook} is set. + */ + public final Duration connectHookTimeout; + private DriverChannelOptions( CqlIdentifier keyspace, List eventTypes, EventCallback eventCallback, String ownerLogPrefix, - boolean reportConfig) { + boolean reportConfig, + ConnectHook connectHook, + Duration connectHookTimeout) { this.keyspace = keyspace; this.eventTypes = eventTypes; this.eventCallback = eventCallback; this.ownerLogPrefix = ownerLogPrefix; this.reportConfig = reportConfig; + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; } public static class Builder { @@ -73,6 +98,8 @@ public static class Builder { private EventCallback eventCallback = null; private String ownerLogPrefix = null; private boolean reportConfig = false; + private ConnectHook connectHook = null; + private Duration connectHookTimeout = null; public Builder withKeyspace(CqlIdentifier keyspace) { this.keyspace = keyspace; @@ -100,9 +127,27 @@ public Builder reportConfig(boolean reportConfig) { return this; } + /** + * Arms a step that vets every candidate channel after protocol initialization, bounded by the + * given timeout. See {@link ConnectHook} for the contract. + */ + public Builder withConnectHook(ConnectHook connectHook, Duration connectHookTimeout) { + Preconditions.checkNotNull(connectHook); + Preconditions.checkNotNull(connectHookTimeout); + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; + return this; + } + public DriverChannelOptions build() { return new DriverChannelOptions( - keyspace, eventTypes, eventCallback, ownerLogPrefix, reportConfig); + keyspace, + eventTypes, + eventCallback, + ownerLogPrefix, + reportConfig, + connectHook, + connectHookTimeout); } } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java index dd7630a6530..042006ea9a6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java @@ -52,7 +52,6 @@ import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,7 +158,6 @@ private enum Step { GET_CLUSTER_NAME, SET_KEYSPACE, AUTH_RESPONSE, - REGISTER, } private class InitRequest extends ChannelHandlerRequest { @@ -170,12 +168,10 @@ private class InitRequest extends ChannelHandlerRequest { private Message request; private Authenticator authenticator; private ByteBuffer authResponseToken; - private final List registerEventTypes; InitRequest(ChannelHandlerContext ctx) { super(ctx, timeoutMillis); this.step = querySupportedOptions ? Step.OPTIONS : Step.STARTUP; - this.registerEventTypes = options.eventTypes; } @Override @@ -206,8 +202,6 @@ Message getRequest() { return request = new Query("USE " + options.keyspace.asCql(false)); case AUTH_RESPONSE: return request = new AuthResponse(authResponseToken); - case REGISTER: - return request = new Register(registerEventTypes); default: throw new AssertionError("unhandled step: " + step); } @@ -330,21 +324,11 @@ void onResponse(Message response) { if (options.keyspace != null) { step = Step.SET_KEYSPACE; send(); - } else if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); } else { setConnectSuccess(); } } } else if (step == Step.SET_KEYSPACE && response instanceof SetKeyspace) { - if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); - } else { - setConnectSuccess(); - } - } else if (step == Step.REGISTER && response instanceof Ready) { setConnectSuccess(); } else if (response instanceof Error) { Error error = (Error) response; @@ -366,17 +350,6 @@ void onResponse(Message response) { } else if (step == Step.SET_KEYSPACE && error.code == ProtocolConstants.ErrorCode.INVALID) { fail(new InvalidKeyspaceException(error.message)); - } else if (step == Step.REGISTER - && error.code == ErrorCode.PROTOCOL_ERROR - && error.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { - // The server rejected CLIENT_ROUTES_CHANGE as an unknown event type. - // Fail the connection so that the caller (ClientRoutesTopologyMonitor.init()) - // gets a clear error instead of silently degrading. - fail( - "Server does not support CLIENT_ROUTES_CHANGE event " - + "(requires ScyllaDB Enterprise >= 2026.1). " - + "Either upgrade the server or remove the client routes configuration.", - null); } else { failOnUnexpected(error); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java index 5b4ff4dcec8..43a3ac412a4 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java @@ -66,7 +66,33 @@ public interface NettyOptions { /** * A hook invoked each time the driver creates a client bootstrap in order to open a channel. This - * is a good place to configure any custom option on the bootstrap. + * is a good place to configure any custom option, attribute, or {@link + * Bootstrap#resolver(io.netty.resolver.AddressResolverGroup)} on the bootstrap. + * + *

The hook runs once per logical connection to a node. When a hostname expands to several IP + * addresses, the same bootstrap is shared by every per-address attempt (each attempt uses a + * {@link Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it); likewise, protocol-version + * downgrade retries reuse it. Before multi-address support the hook ran once per attempt, + * including once per downgrade retry. + * + *

Anything the hook allocates must outlive the call. Because it runs per connection, a + * resolver group constructed inside it — {@code bootstrap.resolver(new + * DnsAddressResolverGroup(...))} — is a fresh group every time: a new {@code DnsNameResolver} + * with its own datagram channel and its own cold cache, plus a listener registered on the I/O + * loop's termination future that only {@code AddressResolverGroup.close()} removes. Build the + * group once, hold it in a field, and hand the same instance to every bootstrap. + * + *

The bootstrap does not carry the driver's channel handler yet, and a handler + * installed by this hook is not honoured: the driver sets its own handler on each + * per-attempt copy afterwards (and logs a one-time warning if it overwrites one). To customize + * the pipeline, use {@link #afterChannelInitialized(Channel)} instead. (Before multi-address + * support the hook ran after the driver's handler was installed, so replacing it was technically + * possible; that was never a supported extension point.) + * + *

An {@link io.netty.channel.EventLoopGroup} set by this hook is likewise not honoured: + * {@code Bootstrap#clone(EventLoopGroup)} assigns the group unconditionally, so each per-attempt + * copy is bound to the loop the driver picked from {@link #ioEventLoopGroup()}. Configure the + * group there instead. */ void afterBootstrapInitialized(Bootstrap bootstrap); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 9da3a2a8aa2..659b5c20893 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -19,7 +19,6 @@ import com.datastax.oss.driver.api.core.AllNodesFailedException; import com.datastax.oss.driver.api.core.AsyncAutoCloseable; -import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.connection.ReconnectionPolicy; @@ -28,6 +27,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.channel.ChannelEvent; +import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.channel.DriverChannel; import com.datastax.oss.driver.internal.core.channel.DriverChannelOptions; import com.datastax.oss.driver.internal.core.channel.EventCallback; @@ -38,13 +38,17 @@ import com.datastax.oss.driver.internal.core.metadata.DefaultTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; +import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metadata.TopologyEvent; +import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; import com.datastax.oss.driver.internal.core.util.Loggers; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.internal.core.util.concurrent.Reconnection; import com.datastax.oss.driver.internal.core.util.concurrent.RunOrSchedule; import com.datastax.oss.driver.internal.core.util.concurrent.UncaughtExceptions; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -54,17 +58,26 @@ import com.datastax.oss.protocol.internal.response.event.StatusChangeEvent; import com.datastax.oss.protocol.internal.response.event.TopologyChangeEvent; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import io.netty.util.concurrent.EventExecutor; +import java.time.Duration; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Queue; +import java.util.Set; +import java.util.UUID; import java.util.WeakHashMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.function.Consumer; import net.jcip.annotations.ThreadSafe; @@ -90,6 +103,24 @@ public class ControlConnection implements EventCallback, AsyncAutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ControlConnection.class); + /** + * How many removed host ids {@code SingleThreaded#removedHostIds} keeps before evicting the + * oldest. Large enough that the set still covers the churn of a rolling restart many times over, + * small enough that it cannot grow into a leak over a session's lifetime. + */ + private static final int MAX_REMOVED_HOST_IDS = 256; + + /** + * How many consecutive reconnection rounds may be refused entirely by exclusions before {@code + * SingleThreaded#removedHostIds} is cleared anyway. + * + *

Small, because every one of those rounds is a round that reached a live server and then + * threw the channel away: there is nothing to learn from repeating it, and the only thing still + * refusing is a judgement the driver cannot re-check while the control connection is down. Not + * one, because the refusal has to actually take effect -- see {@code #reconnect}. + */ + private static final int MAX_ALL_EXCLUDED_ROUNDS = 3; + private final InternalDriverContext context; private final String logPrefix; private final EventExecutor adminExecutor; @@ -267,6 +298,93 @@ private void processClientRoutesChange(Event event) { .fire(new ClientRoutesUpdateEvent(crce.changeType, crce.connectionIds, crce.hostIds)); } + /** + * Whether {@code error} records a node this connection was not allowed to use, rather than one it + * tried and failed to reach. + * + *

Walks the cause chain rather than testing the top-level throwable, because a refusal is + * wrapped once for every layer it travels through and the number of layers depends on where it + * was raised. From the query plan it arrives bare. From the connect hook -- where a contact + * point's host id is now settled, one candidate at a time -- it comes back through a failed stage + * as a {@link CompletionException}, and {@code ChannelFactory#finishCandidate} then wraps that in + * a {@code ConnectionInitException} before the candidate loop ever sees it. Matching on one fixed + * shape would silently classify the deeper one as a connectivity failure, which is the opposite + * of what it is. + * + *

Shared by every reader of a round's error list, which have to agree on what an exclusion + * means -- {@code anyNodeUnreachable} must not count one as having reached something, {@code + * anyNodeExcluded} decides whether the round spends any of the refusal budget, and {@code + * isAuthFailure} must not let one veto the verdict. + */ + private static boolean isExcluded(Throwable error) { + // Bounded rather than unbounded: getCause() is overridable, so a cyclic chain is possible in + // principle, and no legitimate one is anywhere near this deep. + Throwable cause = error; + for (int depth = 0; cause != null && depth < 16; depth++) { + if (cause instanceof ExcludedNodeException) { + return true; + } + Throwable next = cause.getCause(); + cause = (next == cause) ? null : next; + } + return false; + } + + /** + * Whether {@code error} and every failure attached to it are exclusions, i.e. whether "we were + * not allowed to use it" is the whole story for the node that produced it. + * + *

The test to use wherever an exclusion must not be confused with a connection failure, for + * the same reason {@link ChannelFactory#isAuthOnly} exists: one error no longer means one + * address. A contact point expands to every address its name resolves to and reports a single + * failure with the others attached as {@linkplain Throwable#getSuppressed() suppressed}, so a + * name whose addresses went {@code [excluded, refused]} can surface either half depending on + * which one {@code ChannelFactory#surfacedFailure} promotes. Only when no address was reached at + * all is the node's failure genuinely an exclusion. + */ + private static boolean isExclusionOnly(Throwable error) { + if (!isExcluded(error)) { + return false; + } + for (Throwable suppressed : error.getSuppressed()) { + if (!isExcluded(suppressed)) { + return false; + } + } + return true; + } + + /** + * Whether an exclusion appears anywhere in what {@code error} reports -- as the failure itself, + * in its cause chain, or among the failures attached to it. + * + *

The weakest of the three, and the right one for a caller asking whether the round got as far + * as refusing something rather than whether refusing is all it did. A contact point whose + * addresses went {@code [excluded, refused]} surfaces one half or the other depending on {@code + * ChannelFactory#surfacedFailure}, so neither {@link #isExcluded} nor {@link #isExclusionOnly} + * answers that question without depending on which address happened to be tried last. + */ + private static boolean mentionsExclusion(Throwable error) { + if (isExcluded(error)) { + return true; + } + for (Throwable suppressed : error.getSuppressed()) { + if (isExcluded(suppressed)) { + return true; + } + } + return false; + } + + @VisibleForTesting + static class ExcludedNodeException extends IllegalStateException { + private static final long serialVersionUID = 1; + + ExcludedNodeException(String reason) { + super(reason); + } + } + private class SingleThreaded { private final InternalDriverContext context; private final DriverConfig config; @@ -276,12 +394,65 @@ private class SingleThreaded { private boolean closeWasCalled; private final ReconnectionPolicy reconnectionPolicy; private final Reconnection reconnection; - private DriverChannelOptions channelOptions; + // Computed once in init() and kept; the options themselves are built fresh for every connect + // attempt (see buildChannelOptions), so this is the only part of them that lives here. + private ImmutableList eventTypes; private volatile ControlNodeState controlNodeState = ControlNodeState.NONE; // The last events received for each node private final Map lastNodeDistance = new WeakHashMap<>(); private final Map lastNodeState = new WeakHashMap<>(); + /** + * Host ids the topology monitor has removed, for as long as they stay removed. + * + *

The two maps above are keyed on the {@link Node} instance, which cannot answer for a node + * the driver is about to re-create: {@code MetadataManager#registerNode} mints a fresh {@link + * DefaultNode} for a host id absent from metadata, and {@code DefaultNode} overrides neither + * {@code equals} nor {@code hashCode}, so the new instance is in neither map. Removal is + * therefore also recorded by host id, which survives the re-creation -- see {@link + * #exclusionReasonForHostId}. + * + *

An entry is dropped as soon as the host id reports any state, so a node that legitimately + * comes back is not blocked -- but that path runs on node state events, which arrive from a + * metadata refresh and therefore need a working control connection. It cannot un-refuse a node + * while the control connection is down, which is the one moment this set can do harm, so a + * reconnection round that fails against its whole plan clears it outright (see {@link + * #reconnect}). + * + *

Neither of those bounds it on its own, which is why it is also capped at 256 entries, + * evicting the oldest. Unlike {@code lastNodeDistance} and {@code lastNodeState} next to it -- + * {@link java.util.WeakHashMap}s, so a dead {@link Node} takes its entry with it -- this is + * keyed on a {@link UUID} the driver holds strongly, and the state-event path only ever drops + * the id of a node that came back. A host id that never returns, which is every + * decommissioned or replaced node, would otherwise stay for the life of the session: under + * rolling instance replacement each round mints new ids and none of the old ones are ever + * removed. Evicting the oldest is the right way to lose them, since the risk this set guards + * against -- a contact point whose DNS still lists a node the monitor removed -- fades as the + * removal recedes; the cost of an eviction is at worst one connection attempt to a node that is + * gone, which is the behaviour that predates the set entirely. + */ + private final Set removedHostIds = + Collections.newSetFromMap( + new LinkedHashMap() { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_REMOVED_HOST_IDS; + } + }); + + /** + * How many reconnection rounds in a row drained without reaching anything, every node in them + * having been refused instead. + * + *

Counts only consecutive rounds: a round that reached something resets it, and so does a + * successful reconnection. At {@link #MAX_ALL_EXCLUDED_ROUNDS} it clears {@link + * #removedHostIds} and resets, which is the only exit from that set that does not require the + * control connection this class is trying to restore. See {@code #reconnect}. + * + *

Admin-thread confined, like everything else in this class. + */ + private int consecutiveAllExcludedRounds; + private SingleThreaded(InternalDriverContext context) { this.context = context; this.config = context.getConfig(); @@ -320,15 +491,8 @@ private void init( try { boolean listenClientRoutesEvents = context.getTopologyMonitor() instanceof ClientRoutesTopologyMonitor; - ImmutableList eventTypes = - buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); + this.eventTypes = buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); LOG.debug("[{}] Initializing with event types {}", logPrefix, eventTypes); - channelOptions = - DriverChannelOptions.builder() - .withEvents(eventTypes, ControlConnection.this) - .withOwnerLogPrefix(logPrefix + "|control") - .reportConfig(true) - .build(); Queue nodes = context.getLoadBalancingPolicyWrapper().newControlReconnectionQueryPlan(); @@ -377,11 +541,153 @@ private CompletionStage reconnect() { onSuccessfulReconnect(); }, error -> { + // A round that reached nothing at all leaves this judgement unusable, so drop it. + // + // removedHostIds is only ever cleared by a NodeStateEvent, and those arrive from a + // metadata refresh, which needs the very control connection this is trying to restore. + // A host id recorded as removed on stale information -- a node transiently missing from + // a peers table during a restart, say -- would therefore refuse the only reachable + // address for the rest of the session, and the contact-point fallback that exists to + // re-resolve names could never recover from it. That is a permanent deadlock; a + // resurrection is not, since the ids are re-learned from the first successful refresh. + // + // Clearing here does not weaken the protection where it earns its keep. The case it + // guards -- a contact point whose DNS still lists a node the monitor removed -- happens + // while other nodes are reachable, so those rounds succeed and never reach this branch. + // Only a round that failed against every node in its plan does, and at that point the + // driver's view of who was removed is exactly as stale as its view of everything else. + // + // Only when something was genuinely unreachable, though. This branch is also reached by + // a plan that drained without a single connectivity failure, every node in it having + // been refused instead: excluded by distance or state, or turned away by host id -- + // which is this very set doing its job. Nothing is stale about the driver's view then, + // and clearing on it would undo, on the round that just enforced it, the refusal that + // the next round's contact-point fallback would immediately need again. + // + // But not forever. Enforcing a refusal is one thing; enforcing it for the life of the + // session on evidence that can never be rechecked is another, and that is what an + // unbounded version of this would do. Every other way out of the set needs something + // this situation does not have: a NodeStateEvent arrives from a metadata refresh, which + // needs the control connection being restored here, and the LRU cap only evicts after + // MAX_REMOVED_HOST_IDS *further* removals, which likewise cannot be learned. So a round + // that reached a live server and refused it, over and over, is a round that will keep + // producing the identical outcome -- and if the refused host id is the only address the + // plan has, the session never recovers. Give the refusal MAX_ALL_EXCLUDED_ROUNDS rounds + // to matter, then clear and let the next round find out for itself. Being wrong that + // way costs one connect to a node that is gone; being wrong the other way costs the + // session. + // + // What the budget buys back is the removal set, and only that. The other two sources + // #exclusionReason draws on -- a distance event that made the node IGNORED, a state + // event that removed or forced it down -- are just as un-recheckable while the control + // connection is down, and #exclusionReasonForHostId consults them first for any host id + // still in metadata, so clearing here cannot lift them. That is deliberate: those two + // say the driver was *told* not to use this node, where the removal set says the driver + // inferred it and may be out of date. A plan every entry of which is refused on + // distance or state therefore stays refused, which for a local-DC outage behind a + // contact point resolving only to remote-DC nodes means the control connection does + // not come up -- as it did not before this fallback existed, the plan then being + // empty. Lifting those two as well would put the control connection back on a node an + // operator forced down; see + // https://github.com/scylladb/java-driver/issues/1010. + // + // Only a round that actually refused something counts against that budget. A plan that + // was empty to begin with drains through here too, and it is neither of the two cases + // above: it reached nothing and it turned nothing away, so it learned nothing either + // way -- which is precisely why #anyNodeUnreachable declines to clear on it. Letting it + // drive the budget would discard the set on the one kind of evidence both branches + // agree confers no standing, and an empty plan is reachable: turn the contact-point + // fallback off and let the load balancing policy's view go empty. + if (anyNodeUnreachable(error)) { + consecutiveAllExcludedRounds = 0; + removedHostIds.clear(); + } else if (anyNodeExcluded(error) + && ++consecutiveAllExcludedRounds >= MAX_ALL_EXCLUDED_ROUNDS) { + LOG.debug( + "[{}] {} consecutive reconnection rounds were refused outright; " + + "discarding the set of removed host ids so the next round can retry them", + logPrefix, + consecutiveAllExcludedRounds); + consecutiveAllExcludedRounds = 0; + removedHostIds.clear(); + } result.complete(false); }); return result; } + /** + * Whether a failed reconnection round actually failed to reach something, as opposed to + * having had every node in its plan refused. + * + *

Drawn from the errors the round collected, since both outcomes arrive here as the same + * {@link AllNodesFailedException}. A round with no errors at all -- a plan that was empty to + * begin with -- counts as not unreachable: it never tried anything, so it learned + * nothing about who is reachable and has no standing to discard the removal set. + * + *

{@link #mentionsExclusion}, and deliberately the weakest of the three tests. What clearing + * needs is evidence that the driver's view is stale, and a round that refused a node proves + * the opposite: it handshaked with a live server and read its host id. Neither {@link + * #isExcluded} nor {@link #isExclusionOnly} can be that test, because a contact point reports + * one failure for the whole name and {@code ChannelFactory#surfacedFailure} decides which of + * its addresses speaks -- so a name whose addresses went {@code [excluded, refused]} would be + * classified by whichever one happened to be tried last. Asking whether an exclusion is + * mentioned at all is the only reading that does not turn on that. + * + *

The consequence is that one firewalled sibling record no longer discards the refusal its + * neighbour just earned. Such a round still counts as excluded, so {@code + * MAX_ALL_EXCLUDED_ROUNDS} engages and the set is discarded after a few of them -- the recovery + * is delayed, not removed, and the deadlock the clearing exists to break stays broken. + */ + private boolean anyNodeUnreachable(Throwable roundFailure) { + if (!(roundFailure instanceof AllNodesFailedException)) { + return false; + } + for (List nodeErrors : + ((AllNodesFailedException) roundFailure).getAllErrors().values()) { + for (Throwable nodeError : nodeErrors) { + if (!mentionsExclusion(nodeError)) { + return true; + } + } + } + return false; + } + + /** + * Whether a failed reconnection round refused at least one node, as opposed to having had + * nothing to try in the first place. + * + *

The complement of {@link #anyNodeUnreachable} on the branch that matters, not its + * negation: a round can be neither, and an empty plan is exactly that -- so such a round + * neither clears {@code removedHostIds} nor spends any of the budget that eventually will. + * + *

{@link #mentionsExclusion}, the same test its sibling uses, and that is what makes the two + * complements. Reading one throwable deeper on one side than the other leaves a gap between + * them, and a round can fall into it: an exclusion carried only among a node's {@linkplain + * Throwable#getSuppressed() suppressed} failures is not "unreachable" to the sibling and would + * not be "excluded" here either, so neither branch would run. That gap is not hypothetical -- + * {@code ChannelFactory#surfacedFailure} promotes an authentication failure over an exclusion, + * so a contact point whose addresses went {@code [excluded, bad-credentials]} lands in it + * deterministically, on every round, and both the clearing and the give-up counter stall for + * the life of the session. Which is the deadlock {@code MAX_ALL_EXCLUDED_ROUNDS} exists to + * break. + */ + private boolean anyNodeExcluded(Throwable roundFailure) { + if (!(roundFailure instanceof AllNodesFailedException)) { + return false; + } + for (List nodeErrors : + ((AllNodesFailedException) roundFailure).getAllErrors().values()) { + for (Throwable nodeError : nodeErrors) { + if (mentionsExclusion(nodeError)) { + return true; + } + } + } + return false; + } + private void connect( Queue nodes, List> errors, @@ -393,21 +699,70 @@ private void connect( onFailure.accept(AllNodesFailedException.fromErrors(errors)); } else { LOG.debug("[{}] Trying to establish a connection to {}", logPrefix, node); + NodeInfoHolder capturedNodeInfo = new NodeInfoHolder(); context .getChannelFactory() - .connect(node, channelOptions) + .connect(node, buildChannelOptions(node, capturedNodeInfo)) .whenCompleteAsync( (channel, error) -> { try { - NodeDistance lastDistance = lastNodeDistance.get(node); - NodeState lastState = lastNodeState.get(node); + String exclusion = exclusionReason(node); if (error != null) { if (closeWasCalled || initFuture.isCancelled()) { onSuccess.run(); // abort, we don't really care about the result + } else if (isExclusionOnly(error)) { + // Every address of this contact point turned out to be a node this + // connection may not use -- the connect hook refused each one on its host + // id (see #readChannelNodeInfo). Reported exactly as the two exclusions + // that are decided on the admin thread are: at DEBUG, recorded so that + // exhausting the plan this way says why rather than surfacing a bare + // NoNodeAvailableException, and with no controlConnectionFailed event, + // because nothing failed to connect. Routing it through the branch below + // would warn the operator about a deployment that is in fact reachable, + // and would count a refusal as a connection failure in the metrics. + LOG.debug( + "[{}] Every address of {} belongs to a node this connection may not" + + " use, trying next node", + logPrefix, + node, + error); + List> exclusionErrors = + (errors == null) ? new ArrayList<>() : errors; + exclusionErrors.add(new SimpleEntry<>(node, error)); + connect(nodes, exclusionErrors, onSuccess, onFailure); } else { - if (error instanceof AuthenticationException) { + // isAuthOnly, not a bare instanceof: ChannelFactory reports one failure + // per contact point with the other addresses' failures attached as + // suppressed, and it deliberately surfaces an authentication failure over + // transport ones. A name whose records failed [refused, refused, auth] is + // not an authentication problem, and logging it as one would hide that two + // thirds of the deployment is unreachable. + if (ChannelFactory.isAuthOnly(error)) { Loggers.warnWithException( LOG, "[{}] Authentication error", logPrefix, error); + } else if (ChannelFactory.mentionsAuthentication(error)) { + // Mixed [refused, refused, auth]. Not an authentication problem alone -- + // hence the wording -- but still warned unconditionally, as every + // AuthenticationException was before multi-address support. + // advanced.connection.warn-on-init-error mutes unreachable-node noise; + // it is not a switch for "your credentials are wrong". Folding this case + // into the gated branch below would log the only actionable half of the + // failure at DEBUG. + // + // mentionsAuthentication, not `error instanceof AuthenticationException`: + // which failure of the set arrives here is decided by + // ChannelFactory#surfacedFailure, and it ranks a node-wide failure and an + // invalid keyspace *above* an authentication one. So [auth, + // event-type-rejected] surfaces the rejection with the auth failure + // suppressed, and a test on the type of what arrived would send exactly + // the case this branch exists for down the gated path instead. + Loggers.warnWithException( + LOG, + "[{}] Error connecting to {} (authentication failed on some of its" + + " addresses, others failed for other reasons), trying next node", + logPrefix, + node, + error); } else { if (config .getDefaultProfile() @@ -429,7 +784,27 @@ private void connect( List> newErrors = (errors == null) ? new ArrayList<>() : errors; newErrors.add(new SimpleEntry<>(node, error)); - context.getEventBus().fire(ChannelEvent.controlConnectionFailed(node)); + // Contained for the same reason as the channelOpened fire further down, + // and it is the same hazard: EventBus.fire() has no try/catch of its own + // and RunOrSchedule.on(adminExecutor, ..) runs listeners inline when + // already on the admin loop, so a listener that throws would escape into + // the outer catch (Exception) -- which only logs -- and skip the connect() + // below. The round would then never advance and never complete: initFuture + // stays pending (SessionBuilder.build() blocks) or the Reconnection is + // parked in ATTEMPT_IN_PROGRESS for good. Whether the round advances is + // not a listener's to decide, and a Throwable is caught rather than an + // Exception because the outer catch would not stop an Error here either. + try { + context.getEventBus().fire(ChannelEvent.controlConnectionFailed(node)); + } catch (Throwable t) { + Loggers.warnWithException( + LOG, + "[{}] Listener threw while handling controlConnectionFailed for {};" + + " continuing with the next node", + logPrefix, + node, + t); + } connect(nodes, newErrors, onSuccess, onFailure); } } else if (closeWasCalled || initFuture.isCancelled()) { @@ -439,24 +814,24 @@ private void connect( channel); channel.forceClose(); onSuccess.run(); - } else if (lastDistance == NodeDistance.IGNORED) { - LOG.debug( - "[{}] New channel opened ({}) but node became ignored, " - + "closing and trying next node", - logPrefix, - channel); - channel.forceClose(); - connect(nodes, errors, onSuccess, onFailure); - } else if (lastNodeState.containsKey(node) - && (lastState == null /*(removed)*/ - || lastState == NodeState.FORCED_DOWN)) { + } else if (exclusion != null) { LOG.debug( - "[{}] New channel opened ({}) but node was removed or forced down, " - + "closing and trying next node", + "[{}] New channel opened ({}) but {}, closing and trying next node", logPrefix, - channel); + channel, + exclusion); channel.forceClose(); - connect(nodes, errors, onSuccess, onFailure); + // Recorded for the same reason as the post-handshake exclusion below, and + // marked for the same reason: a plan drained entirely by exclusions has to + // report why rather than surface a bare NoNodeAvailableException, and the + // reconnection's failure callback has to be able to tell "everything was + // refused" from "nothing could be reached". No controlConnectionFailed event + // though -- nothing failed to connect. + List> exclusionErrors = + (errors == null) ? new ArrayList<>() : errors; + exclusionErrors.add( + new SimpleEntry<>(node, new ExcludedNodeException(exclusion))); + connect(nodes, exclusionErrors, onSuccess, onFailure); } else { LOG.debug("[{}] New channel opened {}", logPrefix, channel); DriverChannel previousChannel = ControlConnection.this.channel; @@ -469,7 +844,7 @@ private void connect( previousChannel); previousChannel.forceClose(); } - resolveChannelNodeIfNeeded(channel, (DefaultNode) node) + resolveChannelNodeIfNeeded(channel, (DefaultNode) node, capturedNodeInfo) .whenCompleteAsync( (resolvedNode, fetchError) -> { if (fetchError != null) { @@ -500,19 +875,78 @@ private void connect( new Exception("Channel closed during endpoint resolve"))); connect(nodes, newErrors, onSuccess, onFailure); } else { - controlNodeState = new ControlNodeState(resolvedNode, null); - context - .getEventBus() - .fire(ChannelEvent.channelOpened(resolvedNode)); - channel - .closeFuture() - .addListener( - f -> - adminExecutor - .submit( - () -> onChannelClosed(channel, resolvedNode)) - .addListener(UncaughtExceptions::log)); - onSuccess.run(); + // The guards above ran against the node the query plan offered. + // For a contact point appended by the reconnection fallback that + // is an ephemeral instance which is never the subject of a + // distance or state event, so it is never a key in either map and + // those guards cannot have seen anything. Only now, once the + // handshake has said which node actually answered, is there + // something to ask about -- and asking matters, because nothing + // downstream will: an unchanged distance fires no event, so a + // control connection parked on an excluded node stays there. + String resolvedExclusion = exclusionReason(resolvedNode); + if (resolvedExclusion != null) { + LOG.debug( + "[{}] Channel {} turned out to be {}, which {}; " + + "closing and trying next node", + logPrefix, + channel, + resolvedNode, + resolvedExclusion); + controlNodeState = ControlNodeState.NONE; + // Null out before forceClose(), as above, so that + // onChannelClosed() does not start a redundant reconnection on + // top of the connect() retry below. + ControlConnection.this.channel = null; + channel.forceClose(); + // Recorded, so that exhausting the plan this way reports why + // rather than a bare NoNodeAvailableException. No + // controlConnectionFailed event though: nothing failed to + // connect, the node is simply not one we may use -- same as the + // pre-handshake exclusion branch above. + List> newErrors = + (errors == null) ? new ArrayList<>() : errors; + newErrors.add( + new SimpleEntry<>( + resolvedNode, + new ExcludedNodeException(resolvedExclusion))); + connect(nodes, newErrors, onSuccess, onFailure); + } else { + controlNodeState = new ControlNodeState(resolvedNode, null); + // Contained, because this callback is the only thing that + // completes the round and it is not wrapped by the outer + // catch (Exception) above -- that one guards the *outer* + // whenCompleteAsync, a different stack. EventBus.fire() has no + // try/catch of its own and RunOrSchedule.on(adminExecutor, ..) + // runs listeners inline when already on the admin loop, so a + // user NodeStateListener.onUp that throws would escape here, + // skip onSuccess.run(), and leave the Reconnection parked in + // ATTEMPT_IN_PROGRESS for good. The channel is open either + // way; a listener's failure is not the connection's. + try { + context + .getEventBus() + .fire(ChannelEvent.channelOpened(resolvedNode)); + } catch (Throwable t) { + Loggers.warnWithException( + LOG, + "[{}] Listener threw while handling channelOpened for {};" + + " the control connection is up regardless", + logPrefix, + resolvedNode, + t); + } + channel + .closeFuture() + .addListener( + f -> + adminExecutor + .submit( + () -> + onChannelClosed(channel, resolvedNode)) + .addListener(UncaughtExceptions::log)); + onSuccess.run(); + } } }, adminExecutor); @@ -530,32 +964,334 @@ private void connect( } /** - * Resolves the identity of the node at the other end of the channel. For contact point nodes - * (no hostId), queries system.local and registers a new metadata node. For nodes that already - * have a hostId, returns the node as-is. + * Why {@code candidate} must not be used for the control connection -- the load balancing + * policy has excluded it, or a topology event has -- or {@code null} if nothing rules it out. + * + *

Both maps are keyed on the {@link Node} instance and filled only from events, so a node + * that has never been the subject of one is simply absent, and absent means "nothing known + * against it" rather than "fine". That distinction is why this is asked twice per connect: once + * about the node the query plan offered, and again about the node the handshake proved is at + * the other end, which for a contact point is a different instance. + * + *

Keying on the instance means this can only answer for a node the driver still holds. For a + * node it does not -- one the monitor removed, which a contact point can lead back to -- see + * {@link #exclusionReasonForHostId}. */ - private CompletionStage resolveChannelNodeIfNeeded( - DriverChannel channel, DefaultNode node) { - if (node.getHostId() != null) { - return CompletableFuture.completedFuture(node); + private String exclusionReason(Node candidate) { + if (lastNodeDistance.get(candidate) == NodeDistance.IGNORED) { + return "node became ignored"; + } + if (lastNodeState.containsKey(candidate)) { + NodeState state = lastNodeState.get(candidate); + if (state == null /*(removed)*/ || state == NodeState.FORCED_DOWN) { + return "node was removed or forced down"; + } + } + return null; + } + + /** + * Why the node answering under {@code hostId} must not be used for the control connection, or + * {@code null} if nothing rules it out. + * + *

The host-id-keyed counterpart of {@link #exclusionReason}, for the one moment that has to + * be settled before {@code MetadataManager#registerNode}: a contact point whose DNS + * record still lists a node the topology monitor has removed. Registering first would publish + * that node back into {@code Metadata#getNodes()} and fire {@code NodeStateListener#onAdd} for + * it, and the instance-keyed check could not then undo it -- registerNode returns a brand-new + * {@link DefaultNode}, which is in neither event map. That is exactly the resurrection {@code + * TopologyMonitor#reresolvesNodeAddresses()} describes, reachable through the contact-point + * reconnection fallback whatever that flag says. + * + *

A host id the driver still has in metadata is deferred to {@link #exclusionReason}, so + * IGNORED and FORCED_DOWN keep being reported with their own wording. A host id that is in + * neither metadata nor {@link #removedHostIds} is simply new -- a node that joined while the + * driver was disconnected, which the fallback exists to reach -- and is allowed through. + */ + @Nullable + private String exclusionReasonForHostId(@Nullable UUID hostId) { + if (hostId == null) { + // registerNode's own precondition reports this better than a generic exclusion would. + return null; + } + Node known = context.getMetadataManager().getMetadata().getNodes().get(hostId); + if (known != null) { + return exclusionReason(known); } - return context - .getTopologyMonitor() + return removedHostIds.contains(hostId) ? "node was removed" : null; + } + + /** + * {@link #exclusionReasonForHostId} as a plain map, so that a connect hook can ask it from a + * channel's event loop. + * + *

The hook is where a contact point's identity is settled while its remaining addresses are + * still on hand, and refusing there costs one address instead of the whole plan entry -- but + * the state the answer comes from cannot be read there. {@code lastNodeDistance} and {@code + * lastNodeState} are {@link WeakHashMap}s, whose {@code get} expunges cleared entries and so + * writes; {@code removedHostIds} is a plain {@link LinkedHashMap}-backed set the admin thread + * mutates. All three are confined to {@code adminExecutor}, which is where this runs. + * + *

Enumerated by calling {@link #exclusionReasonForHostId} rather than by restating what it + * does, so the two cannot drift: every host id it can answer non-null for is a key of metadata + * or of {@code removedHostIds}, and both are asked. It also keeps no {@link Node} out of the + * weak maps -- the values are strings -- so a snapshot outliving a connect cannot pin a node + * that metadata has dropped. + * + *

Not usually empty, and worth knowing how big it can get: {@link #exclusionReason} answers + * for an {@code IGNORED} node as well as a removed or forced-down one, so with a local + * datacenter configured this is every node of every remote datacenter. That is a steady state + * rather than a transient, and it is what the hook then refuses one address at a time -- the + * cost of settling identity where the remaining addresses are still on hand, paid on the + * contact points whose records point at an excluded node. + */ + private Map excludedHostIds() { + assert adminExecutor.inEventLoop(); + Set candidates = + new HashSet<>(context.getMetadataManager().getMetadata().getNodes().keySet()); + candidates.addAll(removedHostIds); + Map excluded = new HashMap<>(); + for (UUID hostId : candidates) { + String reason = exclusionReasonForHostId(hostId); + if (reason != null) { + excluded.put(hostId, reason); + } + } + return excluded.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(excluded); + } + + /** + * Options for one connect attempt against one query-plan node. Built fresh per attempt rather + * than cached: an attempt against an unidentified node carries a stateful holder that the + * connect hook fills, and overlapping connect chains are reachable -- the initial {@code + * connect()} runs outside {@code Reconnection} (which serializes only its own attempts), and + * {@code reconnectNow()} checks only {@code initWasCalled} -- so nothing stateful may be shared + * between attempts. + */ + private DriverChannelOptions buildChannelOptions(Node node, NodeInfoHolder capturedNodeInfo) { + DriverChannelOptions.Builder builder = + DriverChannelOptions.builder() + .withEvents(eventTypes, ControlConnection.this) + .withOwnerLogPrefix(logPrefix + "|control") + .reportConfig(true); + if (node.getHostId() == null) { + // A contact point: the driver does not yet know which node answers at each of its + // addresses, so the identity read happens through the connect hook, inside the factory's + // candidate loop, where the hostname's other addresses are still on hand and a rejection + // costs one of them. Read after the connect instead, the candidates are already gone and a + // failure writes off the whole plan entry for that round. + // + // Both criteria are applied there: that the node identifies itself at all, and that the + // host id it gives is one this connection may use. The second needs state the hook's thread + // cannot read, so it is snapshotted here, on the admin loop -- see #excludedHostIds. Taken + // per attempt, which is also when the options are built, so it is as current as the attempt + // is; an event landing mid-connect is caught by the second, live check in + // #resolveChannelNodeIfNeeded. + // Doubled, so that the hook's bound is strictly looser than the deadline of the query it + // wraps. The stage this bounds is a system.local read, which AdminRequestHandler already + // times out on CONTROL_CONNECTION_TIMEOUT -- and DefaultTopologyMonitor snapshots that + // option in its constructor while this reads it live, so handing over the same value gives + // the two deadlines no defined order at all. Whichever wins decides what the operator is + // told: "Connect hook timed out" names the wrapper, the inner DriverTimeoutException names + // the query and the node. A margin makes the inner one win, and makes this what it is meant + // to be -- a backstop against a hook that never completes, which a custom TopologyMonitor + // can produce and the built-in one cannot. It also absorbs a runtime reload that lowers the + // option (reference.conf documents it as not runtime-modifiable, but nothing enforces that + // and getDuration genuinely re-reads); without one, a lowered value abandons every + // candidate of every contact point before its read can finish, and the control connection + // can no longer come up through the contact-point fallback at all. + // + // At zero the margin has nothing to scale: doubling gives zero, and ChannelFactory reads a + // non-positive hook timeout as "no timeout", the way every other consumer of a driver + // timeout option does. So an operator who disables this option disables the backstop and + // the deadline of the query it wraps in one stroke -- and unlike the init-query timeout, + // where ProtocolInitHandler reads the same option and so STARTUP fails first, nothing else + // in the connect path reads this one. What is left bounding a candidate that connects and + // then never answers system.local is the heartbeat, and nothing at all if + // advanced.heartbeat.interval is zero too. That is the exposure the read already had before + // it moved behind a hook -- DefaultTopologyMonitor has always taken its query timeout from + // this option -- so it is left alone rather than given a floor that would contradict the + // option's own convention. + Duration hookTimeout = + config + .getDefaultProfile() + .getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT) + .multipliedBy(2); + Map excludedHostIds = excludedHostIds(); + builder.withConnectHook( + channel -> readChannelNodeInfo(channel, capturedNodeInfo, excludedHostIds), + hookTimeout); + } + return builder.build(); + } + + /** + * The connect hook of a contact-point attempt: reads which node answered and channels it + * straight into the attempt's holder, rejecting the candidate when the node cannot identify + * itself, or identifies itself as one this connection may not use. + * + *

Runs on the channel's event loop and touches no {@code SingleThreaded} state: the holder + * is the only thing written, and it is read back on the admin thread only after the connect + * completes. {@code excludedHostIds} was snapshotted on the admin thread when this attempt's + * options were built, precisely so that nothing here has to read the collections it came from. + * + *

Rejecting here rather than after the connect is what confines the cost of an exclusion to + * the one address that hit it. {@code ChannelFactory} does not treat an {@code + * ExcludedNodeException} as node-wide, so the candidate loop moves on to the hostname's next + * address -- which, for a contact point whose DNS still lists a node the monitor removed, is + * quite likely a node it may use. + */ + private CompletionStage readChannelNodeInfo( + DriverChannel channel, NodeInfoHolder capturedNodeInfo, Map excludedHostIds) { + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + // Before the read, not after a rejection. DefaultTopologyMonitor#getChannelNodeInfo warms its + // system.local column projection from the response, and the projection is an *intersection*: + // it can only shrink. Until this hook existed the read only ever ran against the channel the + // control connection kept, so what it learned was by construction the accepted node's. It now + // runs once per candidate address, and the candidate is not known to be kept when the read + // returns -- ChannelFactory can still abandon it on a REGISTER rejection, or on the hook + // timeout, and #resolveChannelNodeIfNeeded re-asks about the node once the channel is open. + // + // Undoing it on each of those paths instead cannot work, and it is worth saying why, because + // it is the obvious shape: none of them can see the projection a *previous* candidate left + // behind, so an address refused after its read would still be narrowing what the next one -- + // accepted -- goes on to report. Two of them are ChannelFactory's and invisible here anyway. + // Clearing first needs none of that: every read becomes a SELECT * that re-learns from + // whoever answered it. It also drops a projection learned from the *previous* control node on + // a reconnect, which would otherwise be applied to a node that need not carry those columns + // at all. What survives is the last candidate to *answer*, which is not automatically the one + // the loop keeps -- an abandoned candidate is not cancelled, so its response can land on + // either side of the accepted one's, or after this attempt has finished with it. Two of those + // three orders are closed, at the two ends: the reset here, and the reset in front of + // #resolveChannelNodeIfNeeded's fallback read. DefaultTopologyMonitor#toLocalNodeInfo spells + // out which one is left and why closing it needs the projection keyed to its channel. + // + // Narrow deliberately: the hook reads system.local and nothing else, so the peer projections + // cannot be what it narrowed, and re-learning them would cost a SELECT * over every peer row. + // The wide #resetColumnCaches stays what a reconnect calls, where the cluster itself may have + // changed. + topologyMonitor.resetLocalColumnCache(); + return topologyMonitor .getChannelNodeInfo(channel) - .thenComposeAsync( + .thenAccept( nodeInfo -> { - EndPoint resolvedEp = nodeInfo.getEndPoint(); - if (resolvedEp != null && !resolvedEp.equals(channel.getEndPoint())) { - channel.setEndPoint(resolvedEp); - LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, resolvedEp); + // Mirrors DefaultTopologyMonitor's own precondition, so that a custom monitor + // cannot smuggle a null past registerNode: rejecting here costs one address, while + // failing in registerNode later would cost the whole plan entry. + Objects.requireNonNull( + nodeInfo.getHostId(), + "Node info is missing its host id; the node may still be bootstrapping"); + String exclusion = excludedHostIds.get(nodeInfo.getHostId()); + if (exclusion != null) { + throw new ExcludedNodeException(exclusion); } - return context.getMetadataManager().registerNode(nodeInfo); - }, - adminExecutor); + capturedNodeInfo.set(channel, nodeInfo); + }); + } + + /** + * Resolves the identity of the node at the other end of the channel. For nodes that already + * have a hostId, returns the node as-is. For a contact point, the connect hook has already read + * {@code system.local} and captured the result (see {@link #readChannelNodeInfo}); this + * registers a new metadata node from it. + */ + private CompletionStage resolveChannelNodeIfNeeded( + DriverChannel channel, DefaultNode node, NodeInfoHolder capturedNodeInfo) { + if (node.getHostId() != null) { + return CompletableFuture.completedFuture(node); + } + NodeInfo captured = capturedNodeInfo.getFor(channel); + // The pairing with the channel is asserted rather than assumed, and a miss falls back to a + // direct read: a ChannelFactory subclass that does not run the connect hook still gets a + // functioning control connection (and the mocked factories in the unit tests exercise this + // same path). The fallback costs one extra round trip, on that path only. + // + // Cleared before that read as well, for the same reason #readChannelNodeInfo clears before + // its own -- and this is the path that most needs it. A miss means the last capture came + // from some other channel, which is precisely what a stranded candidate's late write does, + // so the projection in the cache is the one *its* read warmed. Identifying the node the + // driver is keeping through a projection intersected against one it refused is the whole + // failure this reset exists to prevent. It is also what makes + // TopologyMonitor#resetLocalColumnCache's "before every one of these reads" true rather than + // true of the hook only. + CompletionStage nodeInfoFuture; + if (captured != null) { + nodeInfoFuture = CompletableFuture.completedFuture(captured); + } else { + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + topologyMonitor.resetLocalColumnCache(); + nodeInfoFuture = topologyMonitor.getChannelNodeInfo(channel); + } + return nodeInfoFuture.thenComposeAsync( + nodeInfo -> { + // Asked before registerNode, not after: registration is what publishes a node into + // Metadata#getNodes() and fires NodeStateListener#onAdd, and for a host id the driver + // does not know it *creates* the node. Deciding afterwards would mean resurrecting a + // node the topology monitor has removed and only then refusing it -- and refusing it + // would not even work, since the instance registerNode just minted is not a key in + // either event map (see #exclusionReason). + // + // Asked again, rather than only in the connect hook: the hook goes on a snapshot taken + // before the connect, so a removal or a distance change that landed while it was in + // flight is not in it. This read is live. The hook is what keeps an exclusion from + // costing the whole plan entry; this is what keeps the answer current. + String exclusion = exclusionReasonForHostId(nodeInfo.getHostId()); + if (exclusion != null) { + return CompletableFutures.failedFuture(new ExcludedNodeException(exclusion)); + } + EndPoint resolvedEp = nodeInfo.getEndPoint(); + EndPoint channelEp = channel.getEndPoint(); + // The channel adopts the node's endpoint so that everything reading it afterwards -- + // DefaultTopologyMonitor's localEndPoint on the next refresh, refreshNode's + // control-node + // check, OptionalLocalDcHelper's endpoint fallback -- sees the same instance the node + // holds, rather than the contact point this connection happened to come up through. + // + // Pinned to the address this channel actually reached, though, because the node's own + // endpoint need not name one: SniEndPoint and ClientRoutesEndPoint hand out a *name* by + // design and re-expand it per connect. Adopting such an endpoint unpinned would make + // channel.getEndPoint().resolve() the shared proxy name, which every SniEndPoint in the + // cluster equals -- so #isControlNode's resolve() comparison would answer true for any + // node, and JAVA-2303's self-peer guard (broadcastRpcAddress.equals(localEndPoint + // .resolve())) would stop matching, an unresolved address never equalling a resolved + // one. + // + // pinTo() is a no-op when there is nothing to pin to, and that case is worth naming + // rather than glossing: both implementations decline an unresolved address, as does + // ChannelFactory#pin, whose javadoc explains why -- pinning to a name freezes an + // endpoint on something that must re-expand. The channel can carry such an address + // when the resolver passed the name through, which resolveCandidates deliberately + // allows for a NoopAddressResolverGroup behind a ProxyHandler, or for a custom resolver + // that reports the name already resolved. The adoption then stores the unpinned + // endpoint -- but both failures above are already live in that configuration whatever + // this line does, because they follow from the channel's address being unresolved and + // the channel's own endpoint is equally unpinned. Skipping the adoption would buy none + // of it back and would lose the node identity this exists to carry, so the fix belongs + // where the unresolved address is accepted; deferred there. + // + // The same test as DefaultNode#setEndPoint, and deliberately not equals(): this is + // exactly the mixed unresolved-vs-resolved case (see PinnableEndPoint#sameIdentity). + if (resolvedEp != null + && resolvedEp != channelEp + && !PinnableEndPoint.sameIdentity(resolvedEp, channelEp)) { + EndPoint adopted = + (resolvedEp instanceof PinnableEndPoint) + ? ((PinnableEndPoint) resolvedEp).pinTo(channelEp.resolve()) + : resolvedEp; + channel.setEndPoint(adopted); + LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, adopted); + } + return context.getMetadataManager().registerNode(nodeInfo); + }, + adminExecutor); } private void onSuccessfulReconnect() { assert adminExecutor.inEventLoop(); + // A round got through, so the count of rounds that did not is no longer consecutive. Reset it + // here rather than only on the failure path, so a session that alternates between a refused + // round and a good one never accumulates its way to a spurious clear. + consecutiveAllExcludedRounds = 0; // If reconnectOnFailure was true and we've never connected before, complete the future now to // signal that the initialization is complete. Schema refresh and LBP initialization for the // first connection are handled by the session initialization path (DefaultSession.init), not @@ -687,10 +1423,72 @@ private boolean isControlNode(Node eventNode) { && eventNode.getHostId().equals(state.current.getHostId())) { return true; } - if (state.current == null - && state.pending != null - && Objects.equals(eventNode.getEndPoint(), state.pending.getEndPoint())) { - return true; + if (state.current == null && state.pending != null) { + // Resolution is still in flight, so there is no host id to compare yet and the endpoint is + // all there is to go on. The channel's own endpoint is what to compare against: unlike the + // pending node's, ChannelFactory has bound it to the one address the connection actually + // went to. + DriverChannel pendingChannel = ControlConnection.this.channel; + if (pendingChannel == null) { + return false; + } + EndPoint eventEndPoint = eventNode.getEndPoint(); + EndPoint channelEndPoint = pendingChannel.getEndPoint(); + // Two lookup-free comparisons, because neither shape is covered by the other. + // + // resolve() settles it when both sides hold a concrete address: the event carries a + // metadata + // node whose endpoint is a resolved IP, and the channel's is pinned to the IP it reached. + // This is the case the plain hostname contact point hits, and comparing resolve() results + // rather than the endpoints keeps DefaultEndPoint#equals -- which resolves the unresolved + // side of a mixed comparison, i.e. a blocking DNS lookup on the admin thread, on an + // arbitrary single address (issue #1006) -- off a path that runs for every distance or + // state + // event arriving during a control connect. + if (Objects.equals(eventEndPoint.resolve(), channelEndPoint.resolve())) { + return true; + } + // But resolve() cannot settle it when the endpoint's current address is a *name*, which is + // the permanent state of an SNI proxy address, of a client route, of anything a custom + // AddressTranslator hands over unresolved -- and, now that contact points are kept + // unresolved, of a plain hostname contact point until its node adopts the endpoint built + // from system.local. The event node resolves to that unresolved name while the channel + // resolves to the IP it was pinned to, and an InetSocketAddress carrying an InetAddress + // never equals one that does not. + // + // asMetricPrefix(), not equals(). Both answer this without resolving for the endpoints that + // key their identity on something other than the current address -- SniEndPoint on proxy + + // serverName, ClientRoutesEndPoint on the host id, both of which a Cloud contact point and + // its metadata node share -- but "does not resolve" is a property of equals() that only + // *this driver's* implementations have, and the one that does not (DefaultEndPoint#equals, + // issue #1006) cannot be told apart from a third-party one written the same way. Naming it + // by class, as this did, denylists the single instance of the hazard we happen to ship and + // walks a custom endpoint straight into it -- a blocking lookup on the admin thread, which + // is the one thing this branch exists to prevent. + // + // The prefix has no such caveat: it is contractually a short path-like string, and the + // driver already calls it for every node on every topology refresh through + // PinnableEndPoint#sameIdentity, so a resolving implementation of it is already broken + // elsewhere and more loudly. It also settles the case the class check had to give up on: + // a pinned copy carries the original's prefix by PinnableEndPoint's contract, so a + // hostname contact point's node and the channel pinned to one of its addresses match here + // -- where the old test answered false and left an IGNORED or forced-down control node + // reached through a hostname without the reconnect its callers below exist to trigger. + // + // What the prefix does not settle, and neither did the equals() this replaced, is a + // deployment where unrelated nodes share one: an AddressTranslator that hands back a name + // gives every node it covers the same DefaultEndPoint, and if a contact point is that same + // name then any node's event matches the pending channel here. Both tests collide on + // exactly the same input -- host string plus port -- so this is inherited rather than + // introduced, and the cost is a reconnectNow() that restarts an in-flight control connect + // rather than a wrong answer about identity. Settling it needs the host id, which is what + // the branch above this one uses and what a pending channel does not have yet. + // + // Not the same predicate as sameIdentity, deliberately: that one *also* compares resolve(), + // because a node must not stay on a stale pin. Here the two sides are a node and a channel, + // and their addresses differing by exactly that pin is the normal case. + return eventEndPoint.getClass() == channelEndPoint.getClass() + && eventEndPoint.asMetricPrefix().equals(channelEndPoint.asMetricPrefix()); } return false; } @@ -713,6 +1511,14 @@ && isControlNode(event.node)) { private void onStateEvent(NodeStateEvent event) { assert adminExecutor.inEventLoop(); this.lastNodeState.put(event.node, event.newState); + UUID hostId = event.node.getHostId(); + if (hostId != null) { + if (event.newState == null /*(removed)*/) { + removedHostIds.add(hostId); + } else { + removedHostIds.remove(hostId); + } + } if ((event.newState == null /*(removed)*/ || event.newState == NodeState.FORCED_DOWN) && channel != null && !channel.closeFuture().isDone() @@ -752,22 +1558,100 @@ private void forceClose() { } } - private boolean isAuthFailure(Throwable error) { - if (error instanceof AllNodesFailedException) { - Collection> errors = - ((AllNodesFailedException) error).getAllErrors().values(); - if (errors.isEmpty()) { - return false; - } - for (List nodeErrors : errors) { - for (Throwable nodeError : nodeErrors) { - if (!(nodeError instanceof AuthenticationException)) { - return false; - } + /** + * Whether every contact point failed for the one reason worth telling the operator to go and fix + * their configuration over: bad credentials, everywhere. + * + *

Each entry is tested with {@link ChannelFactory#isAuthOnly} rather than a bare {@code + * instanceof}, because one entry no longer means one address. {@code ChannelFactory} expands a + * contact-point hostname to every address it resolves to and reports a single failure for the + * name, with the other addresses' failures attached as suppressed exceptions. Looking only at the + * top-level throwable would call a name whose records failed {@code [refused, refused, auth]} an + * authentication failure, and claim in the log that authentication is what is wrong with the + * deployment when two thirds of it is unreachable. + */ + @VisibleForTesting + static boolean isAuthFailure(Throwable error) { + if (!(error instanceof AllNodesFailedException)) { + // Anything else carries no per-node breakdown to inspect, so there is nothing here that says + // every contact point rejected the credentials. + return false; + } + Collection> errors = ((AllNodesFailedException) error).getAllErrors().values(); + if (errors.isEmpty()) { + return false; + } + // An excluded node is skipped rather than allowed to veto. It was never asked for credentials, + // so it is no evidence either way -- and letting it answer would hide a genuine credential + // problem behind one node that happened to be IGNORED or forced down. If skipping leaves + // nothing, the round tried nobody and there is no verdict to report. + // + // Only when the exclusion is the whole story for that node, though: a contact point whose + // addresses went [excluded, auth] was asked for credentials, on the address that reached a + // server, and skipping it would drop the only evidence there is. + // + // Which is also why that node is then tested with the two-argument isAuthOnly, passing the + // same exclusion test. The plain one walks the suppressed failures and rejects anything that + // is not an AuthenticationException -- so it would find the excluded sibling, answer false, + // and return false for the whole round. Setting exclusions aside on both sides is what makes + // the skip above mean anything. + boolean anyTried = false; + for (List nodeErrors : errors) { + for (Throwable nodeError : nodeErrors) { + if (isExclusionOnly(nodeError)) { + continue; + } + if (!ChannelFactory.isAuthOnly(nodeError, ControlConnection::isExcluded)) { + return false; } + anyTried = true; + } + } + return anyTried; + } + + /** + * What one contact-point connect attempt learned about the node that answered: filled by the + * connect hook on the channel's event loop, read back on the admin thread once the connect + * completes. One instance per attempt -- overlapping attempts must not share it, which is why + * {@code DriverChannelOptions} are built per attempt. + */ + @VisibleForTesting + static final class NodeInfoHolder { + + /** + * The two values are published as one reference so that a reader can never see one candidate's + * node info paired with another's channel. Two separate volatile fields would not do: the + * factory's candidate loop can leave a stranded hook behind -- an attempt abandoned on the hook + * timeout, whose {@code system.local} response then arrives anyway -- and its late write would + * land in between the accepted candidate's two writes. The admin thread reading in that window + * would take the rejected candidate's node info as the accepted channel's, and the control + * connection would then register the wrong host id and endpoint for the node it is talking to. + * + *

With the pair atomic, that late write merely makes {@link #getFor} miss, which falls back + * to reading {@code system.local} again on the channel that is actually open. + */ + private volatile Capture capture; + + void set(DriverChannel channel, NodeInfo nodeInfo) { + this.capture = new Capture(channel, nodeInfo); + } + + /** The captured info if it came from {@code channel}: the pairing is asserted, not assumed. */ + NodeInfo getFor(DriverChannel channel) { + Capture current = this.capture; + return (current != null && current.channel == channel) ? current.nodeInfo : null; + } + + private static final class Capture { + final DriverChannel channel; + final NodeInfo nodeInfo; + + Capture(DriverChannel channel, NodeInfo nodeInfo) { + this.channel = channel; + this.nodeInfo = nodeInfo; } } - return true; } /** diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java index b93a16a6525..97aab92fff7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java @@ -26,7 +26,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -68,73 +67,34 @@ public OptionalLocalDcHelper( @Override @NonNull public Optional discoverLocalDc(@NonNull Map nodes) { - String localDcStr = context.getLocalDatacenter(profile.getName()); - Optional localDc; - if (localDcStr != null) { - LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); + String localDc = context.getLocalDatacenter(profile.getName()); + if (localDc != null) { + LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { - localDcStr = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); - LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); - } else { - localDc = Optional.empty(); - } - if (localDc.isPresent()) { - checkLocalDatacenterCompatibility( - localDc.get(), context.getMetadataManager().getContactPoints()); - // Also warn if the configured DC doesn't match any node in the cluster - if (!nodes.isEmpty()) { - boolean found = false; - for (Node node : nodes.values()) { - if (localDc.get().equals(node.getDatacenter())) { - found = true; - break; - } - } - if (!found) { - LOG.warn( - "[{}] Configured local DC '{}' does not match any node's datacenter" - + " (available DCs: {}); please verify your configuration", - logPrefix, - localDc.get(), - formatDcs(nodes.values())); - } - } + localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); + LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); } else { LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix); + return Optional.empty(); } - return localDc; - } - - /** - * Checks if the contact points are compatible with the local datacenter specified either through - * configuration, or programmatically. - * - *

The default implementation logs a warning when a contact point reports a datacenter - * different from the local one, and only for the default profile. - * - * @param localDc The local datacenter, as specified in the config, or programmatically. - * @param contactPoints The contact points provided when creating the session. - */ - protected void checkLocalDatacenterCompatibility( - @NonNull String localDc, Set contactPoints) { - if (profile.getName().equals(DriverExecutionProfile.DEFAULT_NAME)) { - Set badContactPoints = new LinkedHashSet<>(); - for (Node node : contactPoints) { - if (!Objects.equals(localDc, node.getDatacenter())) { - badContactPoints.add(node); + if (!nodes.isEmpty()) { + boolean found = false; + for (Node node : nodes.values()) { + if (localDc.equals(node.getDatacenter())) { + found = true; + break; } } - if (!badContactPoints.isEmpty()) { + if (!found) { LOG.warn( - "[{}] You specified {} as the local DC, but some contact points are from a different DC: {}; " - + "please provide the correct local DC, or check your contact points", + "[{}] Configured local DC '{}' does not match any node's datacenter" + + " (available DCs: {}); please verify your configuration", logPrefix, localDc, - formatNodesAndDcs(badContactPoints)); + formatDcs(nodes.values())); } } + return Optional.of(localDc); } /** diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefresh.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefresh.java index ac68b92fef2..cc3bcf9d6e0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefresh.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefresh.java @@ -22,13 +22,19 @@ import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.net.InetSocketAddress; import java.util.Map; +import java.util.Optional; import java.util.UUID; import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ThreadSafe public class AddNodeRefresh extends NodesRefresh { + private static final Logger LOG = LoggerFactory.getLogger(AddNodeRefresh.class); + @VisibleForTesting final NodeInfo newNodeInfo; AddNodeRefresh(NodeInfo newNodeInfo) { @@ -55,12 +61,70 @@ public Result compute( // If a node is restarted after changing its broadcast RPC address, Cassandra considers that // an addition, even though the host_id hasn't changed :( // Update the existing instance and emit an UP event to trigger a pool reconnection. - if (!existing.getEndPoint().equals(newNodeInfo.getEndPoint())) { + // + // Asked of the broadcast RPC address, not of the endpoint. The endpoint is *derived* from + // that address by the configured AddressTranslator, so for a translator that hands back the + // address it was given the two questions are the same one -- but where they differ, the + // endpoint answers wrongly in both directions and the address answers correctly: + // + // - A translator that returns a name (SubnetAddressTranslator does, under its default + // resolve-addresses = false; so do FixedHostNameAddressTranslator and + // Ec2MultiRegionAddressTranslator now) maps every node it covers to the same endpoint, so + // a node that really did move compares equal and its pool is never told. + // - The same address yields two endpoint representations depending on which system table it + // was read from: DefaultTopologyMonitor#connectedNodeEndPoint gives the control node an + // identity of its own, while its peers row gives it the translator's output. A NEW_NODE + // event for the current control node is therefore a comparison between those two forms, + // and it reports a change on every such event even though nothing moved -- copying the + // peers-derived endpoint in, flipping the metric identity and clearing the node's series + // (see DefaultNode#setEndPoint), then flipping back on the next refresh. + // + // Neither EndPoint#equals nor PinnableEndPoint#sameIdentity can separate those: the first + // resolves the unresolved side of a mixed pair, which is a blocking DNS lookup on the admin + // event loop whose answer depends on which address the resolver lists first (issue #1006), + // and the second compares metric identity, which is exactly what the second case changes + // while the addressing stays put. Both system-table addresses are already resolved, so this + // needs no lookup either way. + // + // An absent address on the *existing* node counts as a change, as the endpoint comparison + // did for a node that had never carried one. + // + // What this does not see is the reverse, and it costs more than the endpoint alone: the + // translator's answer changing while the address does not. The endpoint is a function of + // (address, translator), and two of the translators named above re-derive per call -- + // Ec2MultiRegionAddressTranslator from a live PTR lookup, + // FixedHostNameAddressTranslator from config -- and getNewNodeInfo builds the NodeInfo + // through translate() on every event. So an instance replaced behind an unchanged broadcast + // RPC address keeps the old name here and its pool goes on dialling it. Bounded rather than + // permanent: FullNodeListRefresh runs copyInfos over the existing nodes, so the next full + // refresh -- every control-connection reconnect, and every topology-driven one -- picks the + // new endpoint up. + // + // And copyInfos carries more than the endpoint -- datacenter, rack, host id, schema and + // Cassandra version, tokens, extras, broadcast and listen address -- so for such a node none + // of those are refreshed by this event either, and nothing else in this class re-establishes + // them. Bounded by the same full refresh. The endpoint comparison caught that case and lost + // the two above, which are both unconditional; this trade is the deliberate one. + Optional newRpcAddress = newNodeInfo.getBroadcastRpcAddress(); + // Checked rather than asserted, because the get() below is only safe if it holds and an + // assert is not there in production. Always present in practice -- a NEW_NODE event is + // answered from the peers table, and findInPeers builds no NodeInfo without one -- but + // TopologyMonitor#getNewNodeInfo is an extension point, and an absent address would satisfy + // the inequality against a present one and then throw NoSuchElementException out of + // MetadataRefresh#compute, which MetadataManager#apply does not catch. Nothing to update + // towards and nothing to raise an event about, so the refresh is a no-op. + if (!newRpcAddress.isPresent()) { + LOG.warn( + "[{}] Ignoring node addition for {}: the new node info carries no broadcast RPC " + + "address, so there is nothing to compare against the existing node", + context.getSessionName(), + existing); + return new Result(oldMetadata); + } + if (!newRpcAddress.equals(existing.getBroadcastRpcAddress())) { copyInfos(newNodeInfo, ((DefaultNode) existing), context); - assert newNodeInfo.getBroadcastRpcAddress().isPresent(); // always for peer nodes return new Result( - oldMetadata, - ImmutableList.of(TopologyEvent.suggestUp(newNodeInfo.getBroadcastRpcAddress().get()))); + oldMetadata, ImmutableList.of(TopologyEvent.suggestUp(newRpcAddress.get()))); } else { return new Result(oldMetadata); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 15d825b2efc..0443bc8aee9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -20,19 +20,31 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ClientRoutesEndPoint implements PinnableEndPoint { + + private static final Logger LOG = LoggerFactory.getLogger(ClientRoutesEndPoint.class); -public class ClientRoutesEndPoint implements EndPoint { private final UUID hostId; private final ClientRoutesTopologyMonitor topologyMonitor; private final String metricPrefix; @NonNull private final EndPoint fallbackEndPoint; + /** Kept only so that {@link #pinTo(SocketAddress)} can rebuild an identical copy. */ + @Nullable private final InetAddress broadcastInetAddress; + + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}, + * which key off the host id alone. + */ + @Nullable private final InetSocketAddress pinnedAddress; /** * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. @@ -49,12 +61,23 @@ public ClientRoutesEndPoint( @NonNull UUID hostId, @Nullable InetAddress broadcastInetAddress, @NonNull EndPoint fallbackEndPoint) { + this(topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, null); + } + + private ClientRoutesEndPoint( + @NonNull ClientRoutesTopologyMonitor topologyMonitor, + @NonNull UUID hostId, + @Nullable InetAddress broadcastInetAddress, + @NonNull EndPoint fallbackEndPoint, + @Nullable InetSocketAddress pinnedAddress) { this.topologyMonitor = Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); this.hostId = Objects.requireNonNull(hostId, "HOST uuid cannot be null"); this.fallbackEndPoint = Objects.requireNonNull(fallbackEndPoint, "Fallback endpoint cannot be null"); this.metricPrefix = buildMetricPrefix(broadcastInetAddress, hostId); + this.broadcastInetAddress = broadcastInetAddress; + this.pinnedAddress = pinnedAddress; } @NonNull @@ -62,18 +85,135 @@ public UUID getHostId() { return hostId; } + /** + * The endpoint {@link #resolve()} falls back to when this node has no client route. + * + *

Exposed so that {@link ClientRoutesTopologyMonitor#buildNodeEndPoint} can avoid nesting one + * of these inside another: for the {@code system.local} row the superclass hands back the control + * channel's own endpoint, which in a client-routes deployment is already a {@code + * ClientRoutesEndPoint} -- and a pinned one, so nesting it would freeze the fallback on + * one proxy IP and add a level per control reconnect. + */ + @NonNull + EndPoint getFallbackEndPoint() { + return fallbackEndPoint; + } + + /** + * Returns the address connections should be opened to. + * + *

The client route for this host id is an in-memory lookup over the cached {@code + * system.client_routes} contents, and it yields exactly one address by design, so this neither + * blocks nor expands to several candidates. The route's hostname is returned {@linkplain + * InetSocketAddress#isUnresolved() unresolved}: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup}, so a custom resolver is honoured and no DNS lookup + * runs on the caller (the admin event loop, for control-connection reconnects). + * + *

When the topology monitor has no route for this host id — i.e. the node is not reached + * through a cloud private endpoint — this delegates to the fallback endpoint. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly. + */ @NonNull @Override public SocketAddress resolve() { + if (pinnedAddress != null) { + return pinnedAddress; + } + InetSocketAddress address; try { - InetSocketAddress address = topologyMonitor.resolve(hostId); - if (address != null) { - return address; - } - } catch (IOException e) { - throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); + address = topologyMonitor.resolve(hostId); + } catch (IllegalStateException e) { + // The monitor is closed, so its route cache is gone -- but resolve() still has to answer, and + // the honest answer is "no route available", which is what the fallback endpoint is for. + // Throwing here is not contained anywhere useful: PinnableEndPoint#sameIdentity compares + // resolve() results for every node of every topology refresh, and neither NodesRefresh nor + // MetadataManager#apply catches, so a refresh that raced session shutdown would be dropped + // whole and surface only as a DEBUG log in ControlConnection#onSuccessfulReconnect. + // + // Logged rather than swallowed silently: in a private-endpoint deployment the fallback is the + // node's raw broadcast address, which is not client-routable, so the visible symptom is a + // bare + // connect timeout with nothing naming the cause. + LOG.debug( + "[{}] Client routes monitor is closed, falling back to {} for this node", + hostId, + fallbackEndPoint); + address = null; + } + return address != null ? address : fallbackEndPoint.resolve(); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirror DefaultEndPoint: an address we cannot hold in an InetSocketAddress field skips + // pinning rather than failing the connection. So does an unresolved one -- resolve() hands out + // the route's hostname unresolved, and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze the endpoint on + // a name that still re-expands on every connect: no address stability gained, and the route + // lookup silenced for good, since resolve() short-circuits once pinned. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; + } + return new ClientRoutesEndPoint( + topologyMonitor, + hostId, + broadcastInetAddress, + fallbackEndPoint, + (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true} when the address came from a route: a route's addresses are alternative + * ways in to this one node, so connections may be spread across them. + * + *

Otherwise the address is the fallback endpoint's, and whether those addresses are + * interchangeable is not this class's to claim -- for the usual fallback, a {@code + * DefaultEndPoint} built from a translated broadcast address, it is {@code false}, and a + * translator that hands back a name ({@code SubnetAddressTranslator} does, under {@code + * resolve-addresses = false}) is exactly the case where spreading would land one node's channels + * on different hosts. So the question is deferred to whoever owns the address. + * + *

Which of the two it is is read off {@code resolvedAddress} rather than by asking the route + * cache a second time. The cache is an {@code AtomicReference} swapped from the routes-query + * thread, not from the one {@code ChannelFactory#connect} runs on, so a {@code + * CLIENT_ROUTES_CHANGE} landing between {@link #resolve()} and this call would otherwise have the + * two disagree -- and in the direction that matters, a route appearing after a fallback address + * was already chosen, the disagreement authorises shuffling exactly the kind of name this method + * exists to protect. + * + *

Reading the fallback twice in one connect is safe for every fallback the driver builds: + * {@code fallbackEndPoint} is final, and each of them is a {@link DefaultEndPoint} whose {@code + * resolve()} is a field read. It is not safe by type, though -- the field is declared + * {@link EndPoint}, and on the {@code system.local} path it holds whatever {@code + * DefaultTopologyMonitor#buildNodeEndPoint} returned, which passes a subclass's endpoint through + * unchanged. A fallback whose {@code resolve()} is not idempotent -- the shape {@link + * SniEndPoint} itself had until contact points were kept unresolved, rotating through the proxy's + * A-records on every call -- would answer differently here than it did there, and its own address + * would then be reported as route-derived and spread across. Not reachable in tree; closing it + * properly means having {@code resolve()} report which source it used, which is per-connect state + * on an endpoint shared by every connect, so it is named here rather than claimed away. + */ + @Override + public boolean addressesAreInterchangeable(@NonNull SocketAddress resolvedAddress) { + // Pinned: resolve() short-circuits on the pinned address, so that is what was handed out, and + // it denotes the single server this endpoint is now fixed to. Mirrored here so the pinned case + // does not consult the route cache at all -- resolve() has not done so since it was pinned. + if (pinnedAddress != null) { + return false; } - return fallbackEndPoint.resolve(); + // resolve() returns either the route's address or the fallback's, so anything that is not the + // fallback's came from a route. + return !resolvedAddress.equals(fallbackEndPoint.resolve()) + || (fallbackEndPoint instanceof PinnableEndPoint + && ((PinnableEndPoint) fallbackEndPoint).addressesAreInterchangeable(resolvedAddress)); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index 1ffc35fd9f4..177be8d8879 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -21,6 +21,7 @@ import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; @@ -32,9 +33,9 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; @@ -195,9 +196,18 @@ void setResolvedRoutes(Map routes) { resolvedRoutesCache.set(Collections.unmodifiableMap(new HashMap<>(routes))); } + /** + * Returns the client route for {@code hostId} as an {@linkplain InetSocketAddress#isUnresolved() + * unresolved} address, or {@code null} if this node has no route. + * + *

The route's hostname is deliberately left unresolved: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup} at connection time. That keeps this method a pure + * in-memory cache lookup, so it is safe to call from an event loop, and it means a custom + * resolver applies to client routes just like it does to contact points. + */ @Nullable - public InetSocketAddress resolve(@NonNull UUID hostId) - throws IllegalStateException, UnknownHostException { + public InetSocketAddress resolve(@NonNull UUID hostId) throws IllegalStateException { if (closed) { throw new IllegalStateException("Topology monitor is closed"); } @@ -206,7 +216,7 @@ public InetSocketAddress resolve(@NonNull UUID hostId) return null; // no client route for this node — caller falls back to default } - return new InetSocketAddress(resolveAddress(route.getHostname()), route.getPort()); + return InetSocketAddress.createUnresolved(route.getHostname(), route.getPort()); } /** @@ -304,6 +314,19 @@ private CompletionStage executeRefresh( } UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); String address = Objects.requireNonNull(row.getString("address")); + // Emptiness is checked separately from nullity, because isNull() above is + // false for an empty string while ClientRouteRecord's constructor rejects one. + // Like the port range check further down, this is about the diagnostic rather + // than about containment -- the catch around the construction is what keeps one + // bad row from costing the whole refresh -- and it names the column, which the + // constructor's message cannot. + if (address.isEmpty()) { + LOG.error( + "[{}] Skipping client route for host_id={}: address column is empty", + logPrefix, + hostId); + continue; + } // Select port based on SSL configuration at record creation time. // Skip the record if the required port column is absent. @@ -323,6 +346,26 @@ private CompletionStage executeRefresh( useSSL ? "tls_port" : "port"); continue; } + // Range-checked here so that the row is skipped with a diagnostic rather + // than thrown out of this loop: ClientRouteRecord's constructor rejects the same + // range, and its IllegalArgumentException would escape the enclosing + // thenAccept() -- see the catch around the construction below for what that + // costs. The check earns its place by naming the offending column, which the + // constructor cannot. + // + // Zero is rejected here as it is there: it is the "any port" sentinel, never + // something a node can be reached on. + if (effectivePort <= 0 || effectivePort > 65535) { + LOG.error( + "[{}] Skipping client route for host_id={} ({}): " + + "port column ({}) is out of range: {}", + logPrefix, + hostId, + address, + useSSL ? "tls_port" : "port", + effectivePort); + continue; + } // Apply connectionAddr override if configured for this connection_id String connId = @@ -331,12 +374,34 @@ private CompletionStage executeRefresh( : null; if (connId != null) { String override = connectionAddrOverrides.get(connId); + // Not re-validated: ClientRouteProxy's constructor already rejects an empty or + // blank override, one carrying a port, and one with characters no hostname or + // IP address has, so anything in this map is at least as usable as the column + // it replaces. if (override != null) { address = override; } } - newRoutes.put(hostId, new ClientRouteRecord(hostId, address, effectivePort)); + // Guarded even though every argument has just been validated: this loop runs + // inside thenAccept(), so an IllegalArgumentException escaping it skips the + // resolvedRoutesCache.set() below and discards every route parsed in this pass -- + // not just this row. The cache then keeps its previous contents (empty, before + // the first successful refresh) while the offending row stays in the table, so + // every later refresh fails the same way and client routing is off for the whole + // cluster. A backstop for whatever ClientRouteRecord validates next, so that the + // blast radius of one bad row stays one row. + try { + newRoutes.put(hostId, new ClientRouteRecord(hostId, address, effectivePort)); + } catch (IllegalArgumentException e) { + LOG.error( + "[{}] Skipping unusable client route for host_id={} ({}:{}): {}", + logPrefix, + hostId, + address, + effectivePort, + e.getMessage()); + } } if (isTargetedRefresh) { @@ -456,6 +521,22 @@ protected EndPoint buildNodeEndPoint( @NonNull AdminRow row, @Nullable InetSocketAddress broadcastRpcAddress, @NonNull EndPoint localEndPoint) { + EndPoint fallback = super.buildNodeEndPoint(row, broadcastRpcAddress, localEndPoint); + if (fallback instanceof ClientRoutesEndPoint) { + // The system.local row: the superclass hands back the control channel's own endpoint, which + // here is already one of these -- and a pinned one, since ChannelFactory binds the channel's + // endpoint to the address it reached. Nesting it would make this endpoint's route-less + // fallback a frozen proxy IP instead of a static address, and would add one level per control + // reconnect, each retaining a topology monitor and an O(depth) walk in resolve(). Take that + // instance's own fallback, which is the static endpoint the chain is supposed to bottom out + // at. + fallback = ((ClientRoutesEndPoint) fallback).getFallbackEndPoint(); + } + // Unwrapped before the host-id check below, not after it. Returning the superclass's answer + // raw would hand back that same pinned ClientRoutesEndPoint, whose equals/hashCode, metric + // prefix and toString all name the control node -- so a row the driver could not identify + // would come back wearing another node's identity, and resolve() would be frozen on the proxy + // IP that connection reached. UUID hostId = row.getUuid("host_id"); if (hostId == null) { LOG.warn( @@ -464,9 +545,8 @@ protected EndPoint buildNodeEndPoint( + "Falling back to default endpoint resolution.", logPrefix, broadcastRpcAddress); - return super.buildNodeEndPoint(row, broadcastRpcAddress, localEndPoint); + return fallback; } - EndPoint fallback = super.buildNodeEndPoint(row, broadcastRpcAddress, localEndPoint); InetAddress broadcastInetAddress = null; if (broadcastRpcAddress != null) { broadcastInetAddress = broadcastRpcAddress.getAddress(); @@ -480,6 +560,47 @@ protected EndPoint buildNodeEndPoint( return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress, fallback); } + @Override + public boolean reresolvesNodeAddresses() { + // ClientRoutesEndPoint hands the route hostname over unresolved, so the connection layer + // re-expands it on every connection attempt -- but only when a route exists for that host_id + // (see ClientRoutesEndPoint#resolve()); for mixed/incomplete route sets it delegates to a + // static, already-resolved fallback endpoint instead. Only report true when every + // currently-known node actually has a live route; otherwise the contact-point reconnection + // fallback must stay available for the nodes stuck on that fallback. + // + // "Every known node" has to mean at least one: with an empty node set the loop below would + // report true vacuously, suppressing the contact-point fallback at the one moment it is the + // only + // way back -- before the first node refresh, or after the monitor has removed everything. (The + // caller happens to exempt an empty query plan as well, but that is a separate safety net and + // this must not depend on it.) + // + // The answer legitimately changes as route coverage does, so successive reconnection rounds can + // see different values: that tracks reality rather than flapping. The scan is O(nodes) and runs + // once per reconnection attempt, against an in-memory map. + // + // A closed monitor re-resolves nothing at all: resolve() throws IllegalStateException from the + // `closed` guard above, and ClientRoutesEndPoint#resolve() catches that and silently returns + // the static fallback endpoint. Every cached route is therefore inert, so answering from the + // cache alone would keep reporting true and suppress the contact-point fallback for any + // reconnection racing session shutdown. + if (closed) { + return false; + } + Collection nodes = context.getMetadataManager().getMetadata().getNodes().values(); + if (nodes.isEmpty()) { + return false; + } + Map routes = resolvedRoutesCache.get(); + for (Node node : nodes) { + if (!routes.containsKey(node.getHostId())) { + return false; + } + } + return true; + } + /** * Builds the CQL query to fetch client routes. * @@ -645,13 +766,4 @@ public CompletionStage closeAsync() { LOG.debug("[{}] ClientRoutesTopologyMonitor closed", logPrefix); return super.closeAsync(); } - - /** - * Resolves a hostname to an {@link InetAddress}. Extracted as a protected method so that unit - * tests can override it to return stubbed addresses without hitting the network. - */ - @NonNull - protected InetAddress resolveAddress(@NonNull String hostname) throws UnknownHostException { - return InetAddress.getByName(hostname); - } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java index 021824a9b16..5ad96b48dbb 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java @@ -32,7 +32,30 @@ public class CloudTopologyMonitor extends DefaultTopologyMonitor { public CloudTopologyMonitor(InternalDriverContext context, InetSocketAddress cloudProxyAddress) { super(context); - this.cloudProxyAddress = cloudProxyAddress; + // Snapshot the proxy's host string once, here, instead of letting every buildNodeEndPoint() + // re-derive it. SniEndPoint stores the proxy address unresolved, and for a *resolved* input it + // does that by reading getHostString() -- which, when the InetSocketAddress was built from an + // InetAddress rather than from a name, renders that address's mutable, lazily-populated + // hostName field. Anything that calls getHostName() on the instance fills it in, and + // DefaultSslEngineFactory does exactly that under the default allow-dns-reverse-lookup-san, so + // every SniEndPoint built afterwards would get a different equals/hashCode/asMetricPrefix from + // the ones built before. Since this monitor rebuilds every node's endpoint on every topology + // refresh, that moves all of the cluster's per-node metrics at once, mid-session. + // + // Only that spelling drifts, and it is worth being precise about which, so nobody reads this + // guard as redundant and deletes it. new InetSocketAddress("proxy.example.com", 9042) is safe: + // getByName populates the InetAddress's hostName eagerly, so getHostString() answers the same + // string before and after any getHostName() call. new InetSocketAddress( + // InetAddress.getByAddress(bytes), 9042) is the one that moves, from the IP literal to whatever + // the reverse lookup finds. The bundle path never produces either -- CloudConfigFactory + // #getSniProxyAddress already returns createUnresolved(), which the branch below passes + // through untouched -- so what this protects is the programmatic + // SessionBuilder#withCloudProxyAddress, where the caller chooses the constructor. + this.cloudProxyAddress = + cloudProxyAddress.isUnresolved() + ? cloudProxyAddress + : InetSocketAddress.createUnresolved( + cloudProxyAddress.getHostString(), cloudProxyAddress.getPort()); } @NonNull @@ -44,4 +67,14 @@ protected EndPoint buildNodeEndPoint( UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); return new SniEndPoint(cloudProxyAddress, hostId.toString()); } + + @Override + public boolean reresolvesNodeAddresses() { + // Every node is reached through the cloud SNI proxy, and SniEndPoint hands the proxy hostname + // over unresolved, so the connection layer re-expands it on every connection attempt (see + // ChannelFactory#resolveCandidates). Addresses therefore stay current on their own: appending + // the original contact points as a DNS re-resolution fallback would add nothing, and could + // resurrect nodes this monitor has authoritatively removed. + return true; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 7ffbee8e4bb..a02db9bde07 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -18,29 +18,194 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Serializable; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class DefaultEndPoint implements EndPoint, Serializable { +public class DefaultEndPoint implements PinnableEndPoint, Serializable { private static final long serialVersionUID = 1; + private static final Logger LOG = LoggerFactory.getLogger(DefaultEndPoint.class); + + /** Static, so the warning below is emitted once per JVM rather than once per endpoint. */ + @VisibleForTesting + static final AtomicBoolean LOGGED_MIXED_COMPARISON_WARNING = new AtomicBoolean(); + private final InetSocketAddress address; private final String metricPrefix; + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals}, {@link #hashCode} and + * {@link #asMetricPrefix()}: a pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + public DefaultEndPoint(InetSocketAddress address) { + this(address, null); + } + + private DefaultEndPoint(InetSocketAddress address, @Nullable InetSocketAddress pinnedAddress) { this.address = Objects.requireNonNull(address, "address can't be null"); this.metricPrefix = buildMetricPrefix(address); + this.pinnedAddress = pinnedAddress; + } + + /** + * An endpoint identified by {@code identity} but connected to {@code target}: + * {@link #asMetricPrefix()}, {@link #equals} and {@link #toString()} answer for {@code identity}, + * while {@link #resolve()} hands out {@code target}. + * + *

The two differ in their host-name label only — {@code target} is the address a connection + * actually reached, {@code identity} is that same address with the label stripped (see {@code + * AddressUtils#stripHostName}). Splitting them is what lets {@code + * DefaultTopologyMonitor#buildNodeEndPoint} give the connected node an identity of its own + * without changing anything about how connections to it are made: + * + *

+ * + *

{@link #pinTo} cannot express this: a resolved {@code InetSocketAddress}'s equality ignores + * host names, so it would see {@code target} as the address already held and return {@code this}. + */ + static DefaultEndPoint identifiedBy(InetSocketAddress identity, InetSocketAddress target) { + return new DefaultEndPoint(identity, target); } + /** + * Returns the address connections should be opened to: the {@linkplain #pinTo(SocketAddress) + * pinned} one if this is a pinned copy, otherwise the stored address as-is. + * + *

This performs no name resolution. If the stored address is a hostname (i.e. {@linkplain + * InetSocketAddress#isUnresolved() unresolved} — contact points are always kept unresolved, see + * {@link com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}) it is returned + * unresolved, and {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it + * to every IP it maps to through Netty's configured {@code AddressResolverGroup}. Resolving there + * rather than here is deliberate: it keeps any custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized} in the + * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass, and it + * keeps this method non-blocking so it is safe to call from an event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - return address; + return pinnedAddress != null ? pinnedAddress : address; } + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); + if (!(resolvedAddress instanceof InetSocketAddress) + // An unresolved address, as ClientRoutesEndPoint and SniEndPoint also refuse: this endpoint + // hands a hostname over unresolved and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze resolve() on + // a name that must re-expand on every connect. + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress) + // The address we already hold: pinning to it changes nothing, since resolve() and + // toString() would keep yielding what they already do. Returning this rather than an equal + // copy is load-bearing beyond sparing an allocation: {@code + // DefaultTopologyMonitor#connectedNodeEndPoint} keeps the control node's endpoint {@code + // ==} + // to the channel's, which is what lets {@code refreshNode}'s control-node check settle on + // the identity short-circuit in equals() instead of comparing addresses. + || resolvedAddress.equals(this.address)) { + return this; + } + return new DefaultEndPoint(address, (InetSocketAddress) resolvedAddress); + } + + /** + * Whether {@code other} denotes the same node: the stored addresses are compared, ignoring which + * one either endpoint may be {@linkplain #pinTo(SocketAddress) pinned} to. + * + *

Comparing an unresolved name against a resolved address costs a DNS lookup, + * taken inline on the calling thread, because the unresolved side has to be resolved first. It is + * also arbitrary: {@code new InetSocketAddress(name, port)} keeps only the first address + * the name maps to, so for a multi-record name the answer is "equal iff this node is the one the + * resolver happened to list first". And it does not agree with {@link #hashCode()}, which keys on + * the stored address alone -- so a hostname and one of its IPs can be {@code equals} while + * hashing differently, and a hash-based collection of endpoints (the contact-point {@code Set}, + * for one) never treats them as the same entry. + * + *

The first two do not apply when the unresolved side is an IP literal, which is the + * ordinary case rather than an exotic one: contact points are stored unresolved whatever their + * form (see {@code AddressUtils#extract}, which {@code SessionBuilder} always calls with {@code + * resolve = false}), so a plain {@code 1.2.3.4:9042} reaches this branch on every comparison + * against a resolved peer. Re-building it parses the literal with no resolver call and keeps the + * only address it can denote. The warning below is gated on {@link AddressUtils#carriesName} for + * that reason -- it answers name-versus-literal without a lookup, and neither the cost nor the + * arbitrariness is worth reporting for a literal. + * + *

The third hazard does apply to literals, and the gate suppresses the warning for it too. + * {@link #hashCode()} returns {@code address.hashCode()}, which is {@code hostname.hashCode() + + * port} on the unresolved side and {@code addr.hashCode() + port} on the resolved one -- so a + * literal and its resolved twin are {@code equals} while hashing differently, exactly as a + * hostname and one of its IPs are. The consequence named above is reachable that way: {@code + * ContactPoints#merge} de-duplicates through a {@code HashSet} and logs {@code "Duplicate contact + * point"} on a rejected add, so a programmatic resolved {@code 1.2.3.4:9042} alongside a + * config-file {@code "1.2.3.4:9042"} lands in two buckets, is kept twice, and warns about + * nothing. Left as it is with the rest of this method -- see the issue below. + * + *

One more thing the gate gets wrong, in the other direction: {@code carriesName} calls the + * JDK's shorthand literal forms names, because Guava's {@code isInetAddress} requires four dotted + * parts. A contact point written {@code 127.1:9042} works end to end -- {@code + * InetAddress.getAllByName("127.1")} answers {@code /127.0.0.1} -- but the first mixed comparison + * against it logs the warning and burns the once-per-JVM latch, after which the hostname case the + * canary exists for is never reported. {@code ChannelFactory#materializeLiteral} documents the + * same misclassification. Widening the predicate is deferred: it is the shared grammar behind + * {@code reattachHostname} and {@code materializeLiteral}, and those have to keep accepting + * exactly the same strings. + * + *

The branch exists for {@link + * com.datastax.oss.driver.api.core.metadata.Metadata#findNode(EndPoint)}, whose caller may hold + * either form. The two forms do meet, under an {@code AddressTranslator} that hands back a name + * -- {@code SubnetAddressTranslator} does, under its default {@code resolve-addresses = false}, + * and so now do {@code FixedHostNameAddressTranslator} and {@code + * Ec2MultiRegionAddressTranslator}, which had to stop resolving so that a proxy name with several + * A-records fails over -- because the control node's endpoint is resolved while its peers' are + * not. Three driver-internal callers reach it that way, each on a thread worth naming: + * + *

+ * + *

Replacing all three with {@code PinnableEndPoint#sameIdentity}, as this change did wherever + * it controls the comparison, is deferred. It warns once per JVM meanwhile, as a canary for what + * still depends on this -- see https://github.com/scylladb/java-driver/issues/1006. + */ @Override public boolean equals(Object other) { if (other == this) { @@ -48,12 +213,17 @@ public boolean equals(Object other) { } else if (other instanceof DefaultEndPoint) { InetSocketAddress thisAddress = this.address; InetSocketAddress thatAddress = ((DefaultEndPoint) other).address; - // If only one of the addresses is unresolved, resolve the other. Otherwise (both resolved or - // both unresolved), compare as-is. - if (thisAddress.isUnresolved() && !thatAddress.isUnresolved()) { - thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); - } else if (thatAddress.isUnresolved() && !thisAddress.isUnresolved()) { - thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + // If only one of the addresses is unresolved, resolve it. Otherwise (both resolved or both + // unresolved), compare as-is. + if (thisAddress.isUnresolved() != thatAddress.isUnresolved()) { + if (AddressUtils.carriesName(thisAddress.isUnresolved() ? thisAddress : thatAddress)) { + warnAboutMixedComparison(thisAddress, thatAddress); + } + if (thisAddress.isUnresolved()) { + thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); + } else { + thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + } } return thisAddress.equals(thatAddress); } else { @@ -61,6 +231,18 @@ public boolean equals(Object other) { } } + private static void warnAboutMixedComparison(InetSocketAddress one, InetSocketAddress other) { + if (LOGGED_MIXED_COMPARISON_WARNING.compareAndSet(false, true)) { + LOG.warn( + "Compared an unresolved host name against a resolved endpoint address ({} vs {}). This" + + " performs a DNS lookup on the calling thread, only compares the first address the" + + " name maps to, and does not agree with hashCode(); see" + + " https://github.com/scylladb/java-driver/issues/1006. This message is logged once.", + one, + other); + } + } + @Override public int hashCode() { return address.hashCode(); @@ -68,6 +250,9 @@ public int hashCode() { @Override public String toString() { + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which IP a given connection + // landed on is in the channel's own toString(), which Netty builds from the actual remote + // address. return address.toString(); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java index 1b09c26ce16..b2485538d91 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java @@ -102,15 +102,122 @@ public EndPoint getEndPoint() { } public void setEndPoint(@NonNull EndPoint newEndPoint, @NonNull InternalDriverContext context) { - if (!newEndPoint.equals(endPoint)) { - endPoint = newEndPoint; - // metricUpdater is transient, so it can be null on deserialized nodes. - NodeMetricUpdater previousMetricUpdater = metricUpdater; - if (previousMetricUpdater != null - && !(previousMetricUpdater instanceof NoopNodeMetricUpdater)) { - metricUpdater = context.getMetricsFactory().newNodeUpdater(this); - previousMetricUpdater.clearMetrics(); - } + // Nothing downstream can tell the two instances apart, so keep the one already held. Not merely + // an optimization: the instance this node holds may carry a reverse-DNS name cached on its + // InetAddress by an earlier TLS handshake (DefaultSslEngineFactory calls getHostName() under + // the + // default advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true), and every full + // topology refresh mints a brand-new endpoint over a brand-new InetSocketAddress for every + // node. + // Adopting each one unconditionally would throw that name away and make the next connection to + // the node repeat the blocking reverse lookup on a Netty I/O loop -- once per node per refresh + // instead of once per node per session, during exactly the refresh-then-reconnect storms this + // feature exists for. + // + // Deliberately not equals(): see PinnableEndPoint#sameIdentity, which is also what + // ControlConnection uses when it decides whether the control channel should adopt a node's + // endpoint. + if (PinnableEndPoint.sameIdentity(newEndPoint, endPoint)) { + return; + } + + // Metrics are registered under names derived from the endpoint, so they have to be + // re-registered + // whenever those names change -- which is not the same question as whether this is a different + // node. It is narrower in one direction: a PinnableEndPoint copy differs from the original only + // by the address it is pinned to, and both equals() and the metric identity ignore that by + // contract (see PinnableEndPoint). And it is wider in the other: an unresolved hostname and the + // resolved address it maps to compare *equal* (see DefaultEndPoint#equals) while their metric + // prefixes differ, which is exactly what happens when a contact-point node adopts the endpoint + // built from its system.local row. + // + // asMetricPrefix() alone, deliberately, even though the tagging MetricIdGenerator tags metrics + // with the endpoint's toString() rather than its prefix. toString() cannot be used as an + // identity key because it is not stable across equal instances: DefaultEndPoint delegates it to + // InetSocketAddress, which renders InetAddress's *cached* hostName field, and that field is + // populated the first time anything calls getHostName(). DefaultSslEngineFactory does exactly + // that while building an engine, under the default advanced.ssl-engine-factory + // .allow-dns-reverse-lookup-san = true -- on the very instance this node holds, since an + // already-resolved endpoint is passed through unchanged by resolveCandidates() and pin(). So + // after the first channel this node's endpoint renders as "host/1.2.3.4:9042" while the one the + // next refresh decodes from system.peers renders as "/1.2.3.4:9042", and keying on that would + // clear and re-register every node's metrics on every topology refresh -- widening the + // clear/rebuild race described below from "once per endpoint change" to "always". + // + // The cost of leaving it out: a tagging generator can keep reporting under an endpoint string + // the node no longer answers to, until something else changes the prefix. + boolean differentMetricIdentity = + !newEndPoint.asMetricPrefix().equals(endPoint.asMetricPrefix()); + // metricUpdater is transient, so it can be null on deserialized nodes. + NodeMetricUpdater previousMetricUpdater = metricUpdater; + boolean rebuildMetricUpdater = + differentMetricIdentity + && previousMetricUpdater != null + && !(previousMetricUpdater instanceof NoopNodeMetricUpdater); + + // Clearing comes *before* the swap. Dropwizard and MicroProfile do not remember the ids they + // registered under; clearMetrics() recomputes each one from this node's current endpoint (see + // DropwizardMetricUpdater#clearMetrics and MetricIdGenerator#nodeMetricId). Clearing after the + // swap would therefore delete the series the new updater had just registered and leave the old + // ones behind, under a name nothing writes to any more. Micrometer removes the Meter instances + // it holds and does not care either way. + // + // The three steps are not atomic with respect to concurrent metric writes: metricUpdater is + // volatile and read from I/O threads, so a write landing between the clear and the rebuild + // goes through the updater that was just cleared, and Dropwizard re-registers on demand + // (getOrCreateCounterFor -> registry.counter(getMetricId(m))). That resurrects one series, + // named from whichever endpoint this node holds at that instant. + // + // Note the window is narrow but its effect is not transient: the resurrected metric is cached + // in the old updater's map, and ChannelFactory snapshots node.getMetricUpdater() once per + // connection and hands it to the traffic meters, which hold it for the channel's life. So a + // mark that lands here keeps reporting under the old endpoint's name until every channel open + // at that moment has been recycled. Nothing throws -- registry.counter() is get-or-create -- + // and the request path re-reads getMetricUpdater() per request, so the misreporting is + // confined to the byte counters. Closing it properly means having clearMetrics() remove the + // ids it registered under rather than recomputing them from the current endpoint, which is a + // change to every metrics implementation and would also fix a second problem: two nodes can + // briefly share a metric prefix (a control node's endpoint is its contact point's), and then + // this clear deletes the series the other node just registered. + if (rebuildMetricUpdater) { + previousMetricUpdater.clearMetrics(); + } + + // Adopt the newest instance even when it compares equal: a pinned copy carries the address + // every + // subsequent connection to this node will use, so refusing it would freeze the node on the + // first + // address it ever connected to, even after the control connection moved to another one and told + // us about it. (The early return above lets through exactly the instances that differ in that + // address, or in metric identity, or in kind.) + endPoint = newEndPoint; + + // And building comes *after* it: the updaters register every enabled metric from their + // constructor, deriving the names from the endpoint this node holds at that moment. + if (rebuildMetricUpdater) { + NodeMetricUpdater newMetricUpdater = context.getMetricsFactory().newNodeUpdater(this); + // Carry over any pending metrics expiration before publishing the replacement: the factories + // arm and cancel it through node.getMetricUpdater(), so from here on they would only ever + // reach the new one, leaving the old one's timer pending on an object nothing refers to. + // + // Which leaves a window of its own, in the other direction. The hand-over arms the + // replacement while getMetricUpdater() still answers with the old one, so an UP event landing + // in between cancels a countdown that is already gone and misses the live one -- and the node + // comes back healthy with an expiration still ticking, clearing every one of its series an + // hour later with nothing to re-register them until it next goes down and up. The two writers + // really are on different threads: MetadataManager and DropwizardMetricsFactory each take + // their own adminEventExecutorGroup().next(), and advanced.netty.admin-group.size is 2 by + // default. + // + // Ordering cannot close it -- publishing first only moves the window, since a cancel arriving + // before the hand-over is a no-op on an updater that is not armed yet and the hand-over then + // arms it anyway. Only folding the timeout and the expired flag into a single atomic would, + // which is the same conclusion AbstractMetricUpdater#newTimeout reaches about its own + // hand-over race. Reaching this needs an endpoint change to interleave with an UP event + // within a few instructions, on a node that was down with an expiration pending, so the + // window is named rather than claimed away (issue #1010). + newMetricUpdater.adoptExpirationFrom(previousMetricUpdater); + metricUpdater = newMetricUpdater; } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 5a82bfe2c86..fd03b665fa7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -30,6 +30,7 @@ import com.datastax.oss.driver.internal.core.channel.DriverChannel; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.control.ControlConnection; +import com.datastax.oss.driver.internal.core.util.AddressUtils; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; @@ -232,6 +233,11 @@ public void resetColumnCaches() { peersV2Columns = null; } + @Override + public void resetLocalColumnCache() { + localColumns = null; + } + /** * Returns a new list containing only the elements of {@code serverColumns} that are present in * {@code needed}, preserving the server-response order. Returns an empty list (never {@code @@ -352,22 +358,68 @@ public CompletionStage getChannelNodeInfo(DriverChannel channel) { } EndPoint localEndPoint = channel.getEndPoint(); return query(channel, buildQuery(localColumns, "system.local", "key='local'")) - .thenApply( - result -> { - if (localColumns == null && !result.getColumnNames().isEmpty()) { - localColumns = - intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); - } - Iterator iterator = result.iterator(); - if (!iterator.hasNext()) { - throw new IllegalStateException( - "Expected a row in system.local for node info resolution, got empty result"); - } - AdminRow localRow = iterator.next(); - InetSocketAddress broadcastRpcAddress = - getBroadcastRpcAddress(localRow, localEndPoint); - return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); - }); + .thenApply(result -> toLocalNodeInfo(result, localEndPoint)); + } + + /** + * Decodes the single {@code system.local} row of {@code result} into a {@link NodeInfo}, warming + * the local column cache from it on the way. + * + *

The warming is only sound because the caller clears this cache before each read. + * {@link #getChannelNodeInfo} is no longer reached only for the control channel the driver keeps: + * {@code ControlConnection#readChannelNodeInfo} runs it from a connect hook, once per candidate + * address of a contact point, and whether that candidate is the one kept is not known when the + * read returns -- the hook can refuse it, {@code ChannelFactory} can abandon it afterwards (a + * REGISTER rejection, or a hook the timeout gave up on whose response arrives anyway), and {@code + * ControlConnection} re-asks about the node once the channel is open. The intersection can only + * ever shrink, so a refused candidate would otherwise narrow the projection for every {@code + * system.local} read of the session, costing the accepted node's extra columns for as long as the + * cache lives -- and on the first connection, the one round where the hook is guaranteed to run, + * {@code #onSuccessfulReconnect} returns before it would reset them. + * + *

Clearing first rather than undoing afterwards is what makes that hold: an undo on the + * rejection paths cannot see the projection a previous candidate left behind, and two of + * those paths are {@code ChannelFactory}'s and invisible to the hook. With the cache cleared + * first, every read is a {@code SELECT *} that re-learns from whoever answered it. See {@link + * #resetLocalColumnCache()} and {@code ControlConnection#readChannelNodeInfo}. + * + *

The write below is unconditional, not guarded on the cache still being null, because + * "cleared before each read" orders the reads and not the responses. A candidate abandoned + * on the connect-hook timeout is abandoned rather than cancelled, so its {@code system.local} + * answer can arrive after the next candidate cleared the cache -- and a first-writer-wins guard + * would then install the refused candidate's intersection and decline to overwrite it with the + * accepted one's. Overwriting costs nothing on any other path: a projected read returns exactly + * the projection it asked for, and intersecting that with {@code LOCAL_COLUMNS_OF_INTEREST} again + * yields the same list. + * + *

That covers two of the three orders in which a stray answer, the kept candidate's answer, + * and {@code ControlConnection}'s read of the capture can land. A stray answering before + * the kept candidate is overwritten here. A stray answering between the kept candidate and + * that read replaces the capture, so {@code NodeInfoHolder#getFor} misses and {@code + * ControlConnection#resolveChannelNodeIfNeeded} goes back for a fresh read -- cleared first, on + * the channel that is actually open. + * + *

What neither end catches is a stray answering after that read: the capture was still + * the kept candidate's when it was consulted, so nothing falls back, and the stray's warming is + * then the last one to land. The window is the REGISTER round trip plus a hop to the admin + * thread, and a reconnect self-corrects at {@code #resetColumnCaches()}; the first connection + * does not, because {@code ControlConnection#onSuccessfulReconnect} returns at its {@code + * isFirstConnection} check before reaching it. Closing it means the projection carrying the + * channel it was learned from, so that a write from any other channel is dropped outright rather + * than two guards agreeing by construction -- deferred, and named here rather than claimed away. + */ + private NodeInfo toLocalNodeInfo(AdminResult result, EndPoint localEndPoint) { + if (!result.getColumnNames().isEmpty()) { + localColumns = intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); + } + Iterator iterator = result.iterator(); + if (!iterator.hasNext()) { + throw new IllegalStateException( + "Expected a row in system.local for node info resolution, got empty result"); + } + AdminRow localRow = iterator.next(); + InetSocketAddress broadcastRpcAddress = getBroadcastRpcAddress(localRow, localEndPoint); + return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); } @Override @@ -659,8 +711,92 @@ protected EndPoint buildNodeEndPoint( // Don't rely on system.local.rpc_address for the control node, because it mistakenly // reports the normal RPC address instead of the broadcast one (CASSANDRA-11181). We // already know the endpoint anyway since we've just used it to query. + return connectedNodeEndPoint(localEndPoint); + } + } + + /** + * The endpoint to register the connected node under: the address the control channel actually + * reached, rather than the contact point it was reached through. + * + *

They differ when the control connection came up through a contact point, because a contact + * point is kept unresolved and {@code ChannelFactory} binds a {@linkplain PinnableEndPoint + * pinned} copy of it to the one address the channel reached -- a copy that, by that interface's + * contract, is identified exactly like the unpinned original. Registering the node under it would + * give it a hostname identity: metric names and tags derived from a name that denotes the + * whole cluster rather than this node. That is bad enough on its own, but the real damage is that + * the identity is not the node's: the reconnection fallback hands the contact points back on + * every reconnection round, so each successive control node acquires the same one. Two live nodes + * then report under a single metric prefix, sharing get-or-create metric objects, until the next + * refresh moves the older one back to its own address -- and {@code clearMetrics()} recomputes + * the names to delete from the prefix the node still holds, taking the newcomer's freshly + * registered series with it (see {@code DefaultNode#setEndPoint}). + * + *

Deriving the identity from the address actually connected to fixes all of that at once: it + * is this node's own address, so it is unique to it, and re-registering an unchanged control node + * becomes a no-op instead of an identity change. + * + *

The identity comes from the connected address's bytes, not from its host string. That + * string is not the node's either -- for a hostname contact point it is the queried name, which + * every resolver attaches to what it returns and {@code ChannelFactory#reattachHostname} restores + * when a custom one does not, so reading it back here would produce the contact point's prefix + * again and this method would do nothing at all. It is also not stable: for an IP-literal contact + * point it starts out as the literal and begins reporting a reverse-DNS name as soon as {@code + * DefaultSslEngineFactory} calls {@code getHostName()} on the shared {@code InetAddress}, so an + * identity keyed off it would depend on whether TLS is enabled and whether a PTR record exists. + * Stripping the label (see {@link AddressUtils#stripHostName}) settles both. + * + *

What the node connects to is unaffected: the rebuilt endpoint still {@linkplain + * EndPoint#resolve() resolves} to the labelled address the channel reached, so the TLS peer host + * and the Kerberos service name stay the name the operator configured, with no reverse lookup -- + * see {@link DefaultEndPoint#identifiedBy}. + * + *

Two costs come with it, both narrower than the damage above and both named here rather than + * argued away. The rewrite is an identity change, so {@code DefaultNode#setEndPoint} clears the + * previous updater's metrics before the swap, and Dropwizard and MicroProfile recompute the names + * to delete from the prefix the node still holds -- which under a translator that hands back one + * name for the whole cluster is a prefix every node shares, so promoting such a peer to control + * node takes the cluster's node-metric series with it (the root of that is {@code clearMetrics()} + * not remembering what it registered; see https://github.com/scylladb/java-driver/issues/1010). + * And the endpoint this hands back resolves to one address, so the control node is the single + * node in such a deployment that does not get its name re-expanded per connection attempt: + * its pool cannot fail over to a sibling record and will not pick up a DNS change until the + * control connection moves and the node is re-derived from {@code system.peers}. That is the + * trade {@code TopologyMonitor#reresolvesNodeAddresses()} describes for this node, and it is why + * the contact-point reconnection fallback is what recovers it. + * + *

Endpoints this cannot rebuild are returned untouched -- a third-party {@link EndPoint}, or + * one whose {@code resolve()} is not a resolved {@code InetSocketAddress} (the user disabled + * Netty's resolver, or a custom one declined the address, so nothing was pinned). So is one that + * already carries the connected address, which is every reconnection to an identified node and + * every refresh after the first: the existing instance is kept so that the control node's + * endpoint stays {@code ==} to the channel's, which is what lets {@link #refreshNode}'s + * control-node check settle on the identity short-circuit in {@code equals()}. Where that does + * not hold -- the channel kept an unresolved endpoint because adoption was skipped -- the check + * falls through to a full address comparison, which for a name costs a lookup on the admin + * thread and only answers "equal" if the resolver lists the reached address first. A miss there + * is not fatal, but it does send refreshNode on to query the peers table for the control node's + * own address, which by definition has no row; see + * https://github.com/scylladb/java-driver/issues/1006. + */ + private static EndPoint connectedNodeEndPoint(EndPoint localEndPoint) { + if (!(localEndPoint instanceof DefaultEndPoint)) { + return localEndPoint; + } + SocketAddress connected = localEndPoint.resolve(); + if (!(connected instanceof InetSocketAddress) + || ((InetSocketAddress) connected).isUnresolved()) { + return localEndPoint; + } + InetSocketAddress reached = (InetSocketAddress) connected; + InetSocketAddress identity = AddressUtils.stripHostName(reached); + if (identity == null) { return localEndPoint; } + DefaultEndPoint asConnected = DefaultEndPoint.identifiedBy(identity, reached); + return asConnected.asMetricPrefix().equals(localEndPoint.asMetricPrefix()) + ? localEndPoint + : asConnected; } // Called when a new node is being added; the peers table is keyed by broadcast_address, diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index f3f3e4fe346..676b67fc856 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -27,6 +27,8 @@ import com.datastax.oss.driver.api.core.session.Request; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.util.collection.CompositeQueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.internal.core.util.concurrent.ReplayingEventFilter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; @@ -147,7 +149,9 @@ public Queue newQueryPlan( switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: - // The contact points are not stored in the metadata yet: + // The contact points are not stored in the metadata yet. Each unresolved hostname is + // expanded to all its DNS IPs at connection time by ChannelFactory, so one entry per + // contact point is enough here. List nodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(nodes); return new ConcurrentLinkedQueue<>(nodes); @@ -164,20 +168,80 @@ public Queue newQueryPlan( @NonNull public Queue newControlReconnectionQueryPlan() { + // Read the state once, before building the regular plan. State transitions are monotonic + // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value + // newQueryPlan() reads internally; that guarantees we never both build the plan from the + // contact points (pre-RUNNING branch of newQueryPlan) and append them again below. + // + // Note: this is still two separate reads of stateRef (this one, and newQueryPlan()'s own + // internal read a moment later), so a transition landing exactly between them is possible: if + // state flips BEFORE_INIT/DURING_INIT -> RUNNING in that window, newQueryPlan() takes the + // RUNNING branch (a real LBP-built plan) while the state captured here is still pre-RUNNING, + // so the contact-point fallback below is skipped for this one call even though + // regularQueryPlan didn't come from the contact-point branch. This is benign: no crash, no + // duplicate entries, and it self-corrects on the very next reconnection attempt. + // + // Monotonicity leaves the other direction open, and it is worth naming: a RUNNING -> CLOSING + // flip in that same window makes newQueryPlan() take its default branch and return an empty + // plan, which passes both the RUNNING check below and the empty-plan exemption from the + // re-resolving-monitor rule, so the plan handed back is the contact points alone. Also benign: + // ControlConnection abandons a reconnection attempt on closeWasCalled, and every node in that + // plan is one it already had. + State state = stateRef.get(); Queue regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - if (context - .getConfig() - .getDefaultProfile() - .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) { - Set originalNodes = context.getMetadataManager().getContactPoints(); - List contactNodes = new ArrayList<>(); - for (DefaultNode node : originalNodes) { - contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context)); - } + // Only append the contact points as an explicit fallback once the LBP is RUNNING: before that + // (BEFORE_INIT/DURING_INIT), newQueryPlan() above already built regularQueryPlan directly from + // the contact points, so appending them again here would just duplicate every entry. + // + // Skipped when the topology monitor re-resolves node addresses on its own (e.g. proxy-based + // monitors such as client routes or the cloud SNI proxy): those keep addresses fresh without + // this fallback, and appending raw contact points could resurrect nodes the monitor has + // authoritatively removed. The exception is an empty regular plan: with no live node to try, + // reconnection cannot recover on its own, so the contact-point fallback is kept even for those + // monitors. + // + // isEmpty() is asked of a plan a load balancing policy built, which QueryPlan's contract used + // to say the driver never does -- so that contract now names this call, because it is what + // makes the "size() and iterator() never throw" guarantee load-bearing rather than merely + // documented. Nothing cheaper is available: isEmpty() is size() == 0 through + // AbstractCollection, size() reads LazyQueryPlan#getNodes(), and so does poll(), so asking + // through poll() and putting the node back would force the identical computation and allocate + // a wrapper to do it. + if (state == State.RUNNING + && context + .getConfig() + .getDefaultProfile() + .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS) + && (!context.getTopologyMonitor().reresolvesNodeAddresses() + || regularQueryPlan.isEmpty())) { + // Append the original (unresolved) contact points so every IP their hostname resolves to is + // tried as a fallback: ChannelFactory expands each one at connection time, instead of the + // driver being stuck with whatever single IP a metadata node happens to hold. + // + // The retained instances, not fresh copies. MetadataManager holds the contact-point nodes for + // the session's lifetime and the pre-RUNNING branch of newQueryPlan() already hands out these + // very objects, so minting a copy per plan would give each reconnection round a distinct node + // firing its own controlConnectionFailed event -- one set per round, for as long as + // reconnection lasts. Shuffling a fresh list leaves the retained set itself untouched. + // + // Metrics are not a reason either way, and are worth stating because it looks as though they + // should be: DefaultNode.newContactPoint installs NoopNodeMetricUpdater, so a contact-point + // node records nothing, and a fresh copy would be no worse. What that costs is narrower than + // it looks, and narrower than this comment used to claim: errors.connection.auth is written + // in exactly two places, both in ChannelPool#handleError, and a contact-point node never + // reaches a pool -- only this query plan -- so that counter was never written on this path, + // before or after the flip. What is actually missing is every per-node metric for a + // contact-point plan entry; the only thing a reconnect through one reports is + // ChannelEvent.controlConnectionFailed, which NodeStateManager no-ops post-init. Giving these + // nodes real updaters would register metrics under names for ephemeral objects that are + // deliberately absent from metadata, so it is left as is. + List contactNodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(contactNodes); - // Append contact points to the end of the regular query plan so they serve as a fallback - regularQueryPlan.addAll(contactNodes); + // Concatenate rather than mutate: the RUNNING-state regularQueryPlan is a built-in QueryPlan + // whose add()/addAll() throw UnsupportedOperationException (poll() is its only mutator). + // CompositeQueryPlan drains the regular plan first, then the contact-point fallback. + return new CompositeQueryPlan(regularQueryPlan, new SimpleQueryPlan(contactNodes.toArray())); } return regularQueryPlan; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java index cd765c818e6..d8671678306 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java @@ -188,6 +188,12 @@ public boolean wasImplicitContactPoint() { * they are never added to metadata and never exposed to user-facing APIs (events, {@link * com.datastax.oss.driver.api.core.metadata.Metadata#getNodes()}, or {@link * com.datastax.oss.driver.api.core.metadata.NodeStateListener} callbacks). + * + *

The metadata node stores {@code nodeInfo.getEndPoint()} as-is and never re-resolves it on + * its own. Re-resolving the original contact-point hostname to pick up current DNS only happens + * through the original-contact-point reconnection fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}), which re-enters + * the contact points and lets {@code ChannelFactory} expand each hostname at connection time. */ public CompletionStage registerNode(NodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId"); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java new file mode 100644 index 00000000000..00da15b2646 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.SocketAddress; +import java.util.Objects; + +/** + * An {@link EndPoint} that can produce a copy of itself bound ("pinned") to one specific address. + * + *

An endpoint whose hostname maps to several IPs describes a set of candidate addresses, + * but a channel is always connected to exactly one of them. {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} pins the endpoint to the address it + * actually used, and hands the pinned copy to the channel. That matters for two reasons: + * + *

+ * + *

This is an internal extension point: {@code ChannelFactory} pins endpoints that implement it + * and leaves any other implementation untouched, so third-party {@link EndPoint}s keep working + * exactly as before. + * + *

Implementations must keep {@link Object#equals}, {@link Object#hashCode}, {@link + * EndPoint#asMetricPrefix()} and {@link Object#toString()} identical to the unpinned + * original: a pinned copy denotes the same node, and every one of those is part of how the node is + * identified from the outside. Metric names in particular must not change depending on which IP a + * connection happened to land on — and that includes {@code toString()}, which is what {@code + * TaggingMetricIdGenerator} tags node metrics with, and what any third-party {@code + * MetricIdGenerator} is equally free to use. Nodes do adopt pinned copies (see {@code + * DefaultNode#setEndPoint}), so an identity that varied with the pin would silently re-tag a node's + * metrics mid-session. Equality must also stay symmetric: {@code original.equals(pinned)} and + * {@code pinned.equals(original)} must agree, since endpoints are used as set and map keys. + * + *

The pinned address is therefore observable only through {@link EndPoint#resolve()}. That is no + * loss for diagnostics: the address a channel is actually connected to appears in the channel's own + * {@code toString()}, which Netty builds from its remote address, and {@code ChannelFactory} logs + * each candidate as it tries it. + */ +public interface PinnableEndPoint extends EndPoint { + + /** + * Returns a copy of this endpoint that resolves to exactly {@code resolvedAddress}. + * + *

Implementations may return {@code this} when pinning does not apply (for example when the + * address is not of a type they can hold on to), or when it would be a no-op because the endpoint + * already resolves to exactly that address. + * + * @param resolvedAddress the address a connection was successfully established to; must not be + * null and must already be resolved. + */ + @NonNull + EndPoint pinTo(@NonNull SocketAddress resolvedAddress); + + /** + * Whether the addresses this endpoint expands to are interchangeable, i.e. reaching any one of + * them is reaching the same node. + * + *

{@code ChannelFactory} asks this once per connect and answers two questions with it: whether + * it may spread connections across the addresses, and whether a rejection observed at one of them + * settles the rest. The question itself is a property of what the name denotes, and it + * splits the name-based endpoints in two: + * + *

+ * + *

Asked for every endpoint, identified node or contact point alike, but combined with node + * identity differently for each of the two questions {@code ChannelFactory} derives (see its + * {@code spreadAcrossAddresses} and {@code sameServerAtEveryAddress}): + * + *

+ * + *

Which is why {@code false} is the safe default and is what {@link EndPoint} implementations + * outside this interface get: it neither withholds spreading from a contact point nor lets one + * address speak for the others. + * + *

Implementations should derive the answer from {@code resolvedAddress} — what {@link + * #resolve()} just returned for this connect — rather than by asking the same source again. An + * endpoint whose answer comes from mutable state consulted twice can be asked on either side of a + * change and give two answers that describe different addresses, and the one that matters is the + * address actually about to be dialled. {@code ClientRoutesEndPoint} is the case in point: a + * route appearing between the two reads would authorise spreading for an address that came from + * its fallback endpoint, which is the one thing this method exists to prevent. + * + *

Stated as guidance and not as a guarantee, because the driver's own implementation of it + * does not fully hold: {@code ClientRoutesEndPoint#addressesAreInterchangeable} decides "not from + * a route" by comparing against {@code fallbackEndPoint.resolve()}, which is asking a source + * again. It is sound for every fallback the driver builds — each is a {@code DefaultEndPoint} + * whose {@code resolve()} is a field read — and unsound by type, since the field is declared + * {@link EndPoint}. That method's own javadoc sets out the case and the fix it would take (having + * {@code resolve()} report which source it used); it is named here so that an implementer reading + * this paragraph is not told the in-tree code satisfies something it does not. + * + * @param resolvedAddress the address {@link #resolve()} returned for this connect attempt. + */ + default boolean addressesAreInterchangeable(@NonNull SocketAddress resolvedAddress) { + return false; + } + + /** + * Whether two endpoints denote the same node and are indistinguishable to everything that + * reads one: same runtime type, same {@linkplain EndPoint#asMetricPrefix() metric identity}, same + * {@linkplain EndPoint#resolve() current address}. + * + *

Deliberately not {@link Object#equals}: {@code DefaultEndPoint#equals} resolves the + * unresolved side of a mixed comparison, which would put a blocking DNS lookup on the admin + * thread for every endpoint that is still a hostname — and contact points are now kept + * unresolved, so that is reachable (see issue #1006). It is also narrower than {@code equals} in + * one direction and wider in another, which is exactly what callers need: + * + *

+ * + *

The class check keeps a node from staying on a plain fallback endpoint when a dynamic one + * ({@code ClientRoutesEndPoint}) with the same current address arrives. + * + *

{@code toString()} is not part of the test, even though {@code TaggingMetricIdGenerator} + * tags metrics with it rather than with the prefix. It is not stable across equal instances: + * {@code DefaultEndPoint} delegates it to {@code InetSocketAddress}, which renders {@code + * InetAddress}'s cached {@code hostName} field, and that field is populated the first time + * anything calls {@code getHostName()} — which {@code DefaultSslEngineFactory} does while + * building an engine, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying on it would report a + * difference for every node on every topology refresh. The cost of leaving it out: a tagging + * generator can keep reporting under an endpoint string the node no longer answers to, until + * something else changes the prefix. + */ + static boolean sameIdentity(@NonNull EndPoint first, @NonNull EndPoint second) { + return first.getClass() == second.getClass() + && first.asMetricPrefix().equals(second.asMetricPrefix()) + && Objects.equals(first.resolve(), second.resolve()); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index d1ab8eec98d..3199089b00b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -18,61 +18,174 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.shaded.guava.common.primitives.UnsignedBytes; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetAddress; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Comparator; +import java.net.SocketAddress; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -public class SniEndPoint implements EndPoint { - private static final AtomicInteger OFFSET = new AtomicInteger(); +public class SniEndPoint implements PinnableEndPoint { private final InetSocketAddress proxyAddress; private final String serverName; /** - * @param proxyAddress the address of the proxy. If it is {@linkplain - * InetSocketAddress#isUnresolved() unresolved}, each call to {@link #resolve()} will - * re-resolve it, fetch all of its A-records, and if there are more than 1 pick one in a - * round-robin fashion. + * Built once, like {@link DefaultEndPoint}'s and {@link ClientRoutesEndPoint}'s. Both of the + * fields it derives from are final, and {@link PinnableEndPoint#sameIdentity} makes this the + * driver's identity test for endpoints: it is called for both sides of every node on every + * topology refresh, and {@code ControlConnection#isControlNode} calls it again for every distance + * and state event arriving during a control connect. + */ + private final String metricPrefix; + + /** + * The proxy IP this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}: a + * pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + + /** + * @param proxyAddress the address of the proxy. Stored {@linkplain + * InetSocketAddress#isUnresolved() unresolved}, whatever form it was supplied in, so that the + * driver expands a proxy hostname to all of its A-records at connection time and tries each + * of them — see {@link #storeUnresolved}. * @param serverName the SNI server name. In the context of Cloud, this is the string * representation of the host id. */ public SniEndPoint(InetSocketAddress proxyAddress, String serverName) { - this.proxyAddress = Objects.requireNonNull(proxyAddress, "SNI address cannot be null"); + this(proxyAddress, serverName, null); + } + + private SniEndPoint( + InetSocketAddress proxyAddress, + String serverName, + @Nullable InetSocketAddress pinnedAddress) { + this.proxyAddress = + storeUnresolved(Objects.requireNonNull(proxyAddress, "SNI address cannot be null")); this.serverName = Objects.requireNonNull(serverName, "SNI Server name cannot be null"); + this.pinnedAddress = pinnedAddress; + String hostString = this.proxyAddress.getHostString(); + if (hostString == null) { + throw new IllegalArgumentException( + "Could not extract a host string from provided proxy address " + proxyAddress); + } + this.metricPrefix = + hostString.replace('.', '_') + ':' + this.proxyAddress.getPort() + '_' + serverName; + } + + /** + * Stores the proxy address unresolved, whatever form it arrived in. + * + *

{@link #resolve()} hands the stored address to the connection layer as-is, and only an + * unresolved one gets expanded and re-expanded there. A proxy hostname supplied already resolved + * would therefore stay bound to whichever single IP its lookup happened to return, for the life + * of the session: no spreading across the proxy's A-records, no fallback when that one IP stops + * answering, and no pick-up of a DNS change. That is a real possibility for a hostname handed to + * {@link + * com.datastax.oss.driver.api.core.session.SessionBuilder#withCloudProxyAddress(InetSocketAddress)}, + * because the ordinary {@code InetSocketAddress(String, int)} constructor resolves eagerly. + * ({@code CloudConfigFactory}, the usual path, already builds an unresolved address.) + * + *

An address that is already an IP literal is stored unresolved too, even though it has + * nothing to expand, because that is what makes this endpoint's identity stable. A + * resolved address's {@code getHostString()} is not fixed: it starts out as the IP literal and + * begins reporting the reverse-DNS name as soon as anything calls {@code getHostName()} on the + * underlying {@code InetAddress} — which {@code SniSslEngineFactory#newSslEngine} does, on this + * very instance, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying {@link #equals} and + * {@link #asMetricPrefix()} off a string that can change underneath them would move a node's + * metrics mid-session and make endpoints built before and after the first TLS handshake compare + * unequal. An unresolved address has no such field to fill in: its host string is fixed at + * construction, and {@code getHostName()} on it performs no lookup. + * + *

The reverse lookup does not simply move to the {@linkplain #pinTo(SocketAddress) pinned} + * copy, though: it stops happening. {@code ChannelFactory#reattachHostname} labels every + * candidate before it is pinned -- with the queried name when the proxy is a hostname, and with + * the literal itself when it is an IP literal, deliberately, so that the SSL engine is never + * handed a PTR record. Either way {@code getHostName()} on the pinned copy is a field read, so + * {@code advanced.ssl-engine-factory.allow-dns-reverse-lookup-san} no longer changes what this + * endpoint's certificate is validated against. For a hostname proxy that is what happened before + * as well ({@code InetAddress.getAllByName} labels its answers with the queried name). For an + * IP-literal proxy it is a change: those answers carried no label, so the option did reach a PTR + * record, and a proxy certificate that carries only a DNS SAN for that name now fails the + * handshake. Documented in the upgrade guide; not restored, because restoring it means a blocking + * reverse lookup inside the channel initializer, which is what this change removed. + * + *

What this cannot defend against is an instance the caller polluted before handing it over, + * i.e. called {@code getHostName()} on themselves. + * + *

Normalizing here rather than at the call site keeps every {@code SniEndPoint} built from the + * same proxy comparable — {@link #equals} keys on this field — and matches what this endpoint did + * before resolution moved to the connection layer, when it re-resolved the proxy hostname on + * every {@code resolve()} call. + */ + private static InetSocketAddress storeUnresolved(InetSocketAddress proxyAddress) { + return proxyAddress.isUnresolved() + ? proxyAddress + : InetSocketAddress.createUnresolved(proxyAddress.getHostString(), proxyAddress.getPort()); } public String getServerName() { return serverName; } + /** + * Returns the proxy address connections should be opened to. + * + *

Unpinned, this is the stored proxy address as-is — always unresolved (see {@link + * #storeUnresolved}), which {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} + * expands to every proxy A-record, trying each in turn — so a single unreachable proxy IP no + * longer fails the connection. Re-resolving here instead would block whichever event loop called + * us, and would bypass a custom Netty resolver. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns that one proxy IP. That is what + * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} sees: it + * runs inside Netty's channel initializer, so it gets the exact IP the channel is connected to + * without a lookup on the event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - try { - InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); - if (aRecords.length == 0) { - // Probably never happens, but the JDK docs don't explicitly say so - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName()); - } - // The order of the returned address is unspecified. Sort by IP to make sure we get a true - // round-robin - Arrays.sort(aRecords, IP_COMPARATOR); - int index = - (aRecords.length == 1) - ? 0 - : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; - return new InetSocketAddress(aRecords[index], proxyAddress.getPort()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName(), e); + return pinnedAddress != null ? pinnedAddress : proxyAddress; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirrors DefaultEndPoint and ClientRoutesEndPoint: an address this endpoint cannot hold in an + // InetSocketAddress field skips pinning rather than failing the connection, and so does an + // unresolved one. resolve() hands the proxy address over unresolved, and ChannelFactory passes + // it straight back when the user disabled the resolver or a custom one declines it; pinning + // that would freeze this endpoint on a name that must re-expand on every connect -- no address + // stability gained, and the proxy's A-record fallback silenced for good. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; } + return new SniEndPoint(proxyAddress, serverName, (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true}: the proxy routes by server name, so every one of its A-records reaches this + * same node, and connections may be spread across them. That restores what this endpoint did + * itself before resolution moved to the connection layer, when {@code resolve()} sorted the proxy + * A-records and rotated through them on every call. + * + *

It is the right answer to the other question this decides too: because one server answers + * behind every A-record, a protocol-version or event-type rejection observed at one of them is a + * rejection by all of them, and the remaining proxy IPs need not be dialled to confirm it. + * + *

The address is not consulted: there is only ever one source here — the proxy — so every + * address this endpoint can hand out has the same answer. + */ + @Override + public boolean addressesAreInterchangeable(@NonNull SocketAddress resolvedAddress) { + return true; } @Override @@ -94,26 +207,15 @@ public int hashCode() { @Override public String toString() { - // Note that this uses the original proxy address, so if there are multiple A-records it won't - // show which one was selected. If that turns out to be a problem for debugging, we might need - // to store the result of resolve() in Connection and log that instead of the endpoint. - return proxyAddress.toString() + ":" + serverName; + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which proxy IP a given + // connection landed on is in the channel's own toString(), which Netty builds from the actual + // remote address. + return proxyAddress + ":" + serverName; } @NonNull @Override public String asMetricPrefix() { - String hostString = proxyAddress.getHostString(); - if (hostString == null) { - throw new IllegalArgumentException( - "Could not extract a host string from provided proxy address " + proxyAddress); - } - return hostString.replace('.', '_') + ':' + proxyAddress.getPort() + '_' + serverName; + return metricPrefix; } - - @SuppressWarnings("UnnecessaryLambda") - private static final Comparator IP_COMPARATOR = - (InetAddress address1, InetAddress address2) -> - UnsignedBytes.lexicographicalComparator() - .compare(address1.getAddress(), address2.getAddress()); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index 1bb8e343d96..d57c602b986 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -115,8 +115,36 @@ public interface TopologyMonitor extends AsyncAutoCloseable { /** * Resolves the full identity and metadata of the node at the other end of the given channel by - * querying system.local. This is used by the control connection after establishing a channel to - * resolve the contact point's full identity (hostId, datacenter, rack, endpoint, etc.). + * querying system.local. This is used by the control connection to resolve the contact point's + * full identity (hostId, datacenter, rack, endpoint, etc.). + * + *

The control connection calls this from a {@code ConnectHook}, which constrains + * implementations in three ways: + * + *

+ * + *

It is also called directly, off the admin executor, as a fallback for a {@code + * ChannelFactory} that does not run the hook. Implementations therefore have to satisfy the + * stricter of the two, which is the list above. * * @param channel the channel to query system.local on. * @return a future that completes with the resolved node info. @@ -141,4 +169,63 @@ public interface TopologyMonitor extends AsyncAutoCloseable { * {@link DefaultTopologyMonitor}) should override this method. */ default void resetColumnCaches() {} + + /** + * Resets only what {@link #getChannelNodeInfo} can have learned, leaving anything learned from + * the peer tables alone. + * + *

The connect-hook counterpart of {@link #resetColumnCaches()}, called before each candidate's + * read so that what the cache ends up holding belongs to the candidate the driver keeps. Narrow + * because it runs on every such read: the hook only ever queries {@code system.local}, so that is + * the only projection its answer can narrow, and clearing the peer caches with it would cost a + * {@code SELECT *} over every peer row -- every column of every node, token sets included -- once + * per connection attempt. + * + *

{@link #resetColumnCaches()} stays what a reconnect calls: there, the cluster itself + * may have changed while the driver was away, so no projection is trustworthy. + * + *

The default implementation is a no-op, for the same reason as {@link #resetColumnCaches()}. + */ + default void resetLocalColumnCache() {} + + /** + * Whether this monitor re-resolves node addresses dynamically on every connection attempt (for + * example by re-resolving a proxy hostname each time), rather than relying on an endpoint address + * captured once at node-registration time. + * + *

When this returns {@code true}, the control connection's reconnection query plan must not + * append the original contact points as a DNS re-resolution fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}): the monitor + * already keeps addresses fresh, and appending raw contact points could resurrect nodes that the + * monitor has authoritatively removed. + * + *

The default implementation returns {@code false}, which is correct for {@link + * DefaultTopologyMonitor}: the peer nodes it registers hold a {@code DefaultEndPoint} built from + * the broadcast RPC address in {@code system.peers}, an already-resolved physical IP that never + * needs re-resolving. + * + *

Unless the configured {@code AddressTranslator} hands back a name -- {@code + * SubnetAddressTranslator} does, since its {@code resolve-addresses} option defaults to {@code + * false}. Such a peer endpoint is re-expanded per connection attempt by {@code + * ChannelFactory}, and if that name maps to more than one host, one {@code Node}'s connections + * can land on different ones while routing, shard awareness and per-node metrics all attribute + * them to that single node. The candidate loop keeps such addresses in resolver order rather than + * shuffling them -- not because the node is identified, but because {@code DefaultEndPoint} + * reports its addresses as not interchangeable (see {@code + * PinnableEndPoint#addressesAreInterchangeable()} and {@code ChannelFactory#shuffleAndLimit}) -- + * so a pool stays on one host in practice, but the driver has no way to verify the premise. That + * is a property of the translator's output, not of this monitor, so it does not change what this + * flag reports. + * + *

The connected node's own {@code EndPoint} is a different case again. It originates from the + * contact point the control connection used, and {@code ChannelFactory} binds it to the single + * address that connection reached (see {@code PinnableEndPoint}), so it does not re-expand + * on later connection attempts. Recovering from an address change for that node therefore depends + * on this flag being {@code false}, i.e. on the contact-point fallback described above. + * + *

Proxy-based monitors that re-resolve per call should override this to return {@code true}. + */ + default boolean reresolvesNodeAddresses() { + return false; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java index 3d7dc50a7c0..bb510f07dde 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java @@ -29,9 +29,12 @@ import com.datastax.oss.driver.internal.core.session.RequestProcessor; import com.datastax.oss.driver.internal.core.session.throttling.ConcurrencyLimitingRequestThrottler; import com.datastax.oss.driver.internal.core.session.throttling.RateLimitingRequestThrottler; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.cache.Cache; import edu.umd.cs.findbugs.annotations.Nullable; import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; import java.time.Duration; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -49,7 +52,23 @@ public abstract class AbstractMetricUpdater implements MetricUpdater enabledMetrics; + /** + * Stands in for "the expiration already ran" in {@link #metricsExpirationTimeoutRef}, which would + * otherwise be unable to say so: {@code null} means both "nothing armed yet" and "the task has + * cleared the metrics", and {@link #adoptExpirationFrom} has to tell a live node with no + * countdown from a node whose metrics have already expired. + * + *

A sentinel rather than a second field, because the two have to be read and written as one. + * The task's own hand-over used to be a cancel followed by a flag, and a cancel arriving from the + * node's UP handler in between left the flag latched on a live node -- after which the + * next endpoint change armed a fresh hour-long countdown that cleared a healthy node's whole + * series, with nothing to re-register it. One reference makes both transitions a single + * compare-and-set, so a cancel and an expiry can no longer both appear to have won. + */ + @VisibleForTesting static final Timeout EXPIRED = new ExpiredSentinel(); + private final AtomicReference metricsExpirationTimeoutRef = new AtomicReference<>(); + private final Duration expireAfter; protected AbstractMetricUpdater(InternalDriverContext context, Set enabledMetrics) { @@ -148,25 +167,119 @@ protected int orphanedStreamIds(Node node) { } protected void startMetricsExpirationTimeout() { - metricsExpirationTimeoutRef.accumulateAndGet( - newTimeout(), - (current, update) -> { - if (current == null) { - return update; - } else { - update.cancel(); - return current; - } - }); + Timeout mine = newTimeout(); + // A spent expiration is not an armed one: re-arming over EXPIRED is what a node going down + // again after its metrics expired needs, and what adoptExpirationFrom relies on. + // + // The accumulator has to stay a pure function of its two arguments. AtomicReference re-applies + // it when its compare-and-set loses, and the losing read is exactly the one that changes which + // branch it takes: a first pass that saw a live timeout, cancelled the new one and kept the old + // is re-run against a reference the timer task has since set to EXPIRED (or an UP-triggered + // cancel has cleared), and now returns the timeout it just cancelled. Storing that leaves a + // handle that will never fire and is neither null nor EXPIRED, so every later arm preserves it + // and cancels the fresh one instead -- the node's metrics stop expiring until it next comes up. + // Deciding afterwards, from the value the loop settled on, cannot get that wrong. + Timeout winner = + metricsExpirationTimeoutRef.accumulateAndGet(mine, AbstractMetricUpdater::keepArmed); + if (winner != mine) { + mine.cancel(); + } + } + + /** + * The accumulator {@link #startMetricsExpirationTimeout()} hands to {@link AtomicReference}: keep + * whatever countdown is already armed, and take the candidate only when none is. + * + *

Split out and named so that the property its caller cannot demonstrate has somewhere to be + * asserted -- that it is a pure function of its two arguments, and in particular that it does not + * cancel the candidate it declines. See the comment at the call site for what happens when it + * does. + */ + @VisibleForTesting + static Timeout keepArmed(Timeout current, Timeout candidate) { + return (current == null || current == EXPIRED) ? candidate : current; } protected void cancelMetricsExpirationTimeout() { + // Called when the node comes back up, so whatever expiry happened is spent: there is nothing + // left for a later adoptExpirationFrom() to carry over. Clearing the reference says both of + // those at once, which is the point of the sentinel -- an expiry landing either side of this + // is ordered against it rather than racing a separate flag. Timeout t = metricsExpirationTimeoutRef.getAndSet(null); - if (t != null) { + if (t != null && t != EXPIRED) { t.cancel(); } } + /** + * Moves a pending expiration from the updater being replaced onto this one. See {@link + * NodeMetricUpdater#adoptExpirationFrom}. + * + *

Re-armed rather than handed over as-is, so the replacement's own {@code + * startMetricsExpirationTimeout()} runs and the countdown belongs to the object whose metrics it + * will clear. That restarts the clock; expiry is a coarse, hour-scale cleanup and the node has to + * stay down for the whole period either way, so the reset is not worth carrying the original + * deadline around for. + * + *

An expiration that has already run is carried over too, not just a pending one -- and + * so is one that is running right now, which is neither, and which is why the test is that + * the reference was non-null rather than anything {@code cancel()} reports. That is not + * redundant: the replacement's constructor eagerly re-registers the node's whole metric set, so a + * node that expired while down and then had its endpoint change comes back with every series + * present again and, if nothing were armed here, no countdown to clear them. The only other + * caller of {@code startMetricsExpirationTimeout()} is the metrics factory's + * DOWN/FORCED_DOWN/removed handler, and a node that is already down produces no such event -- so + * "nothing armed" would mean the resurrected series outlive the node until it next comes up or is + * removed, which may be never. + * + *

A node that is merely live is not caught by that: {@code + * cancelMetricsExpirationTimeout()}, which the same handler calls on UP, clears the reference, so + * an endpoint change on a healthy node still arms nothing. That holds because the expiry task and + * that cancel contend for one reference -- see {@link #EXPIRED}. It did not while they were a + * cancel followed by a separate flag, and the direction that lost was this one: a UP-triggered + * cancel landing inside the task left a live node looking expired, and this method then armed a + * countdown that cleared a healthy node's metrics an hour later for good. + * + *

Re-arming is best effort. {@code HashedWheelTimer.newTimeout} throws once the timer has been + * stopped ({@code NettyOptions#onClose}) or its pending-task ceiling is reached, and the only + * caller is {@code DefaultNode#setEndPoint} -- reached from {@code NodesRefresh#copyInfos} inside + * {@code MetadataManager}'s apply step, which neither catches nor contains throwables. Letting + * one out would drop an entire metadata refresh, surfacing only as a DEBUG line, over an + * hour-scale cleanup countdown. Losing the countdown costs at worst one node's metrics not + * expiring. + */ + public void adoptExpirationFrom(NodeMetricUpdater previous) { + if (!(previous instanceof AbstractMetricUpdater)) { + return; + } + AbstractMetricUpdater replaced = (AbstractMetricUpdater) previous; + Timeout pending = replaced.metricsExpirationTimeoutRef.getAndSet(null); + if (pending == null) { + return; + } + // Any non-null value is a countdown to carry, whatever cancel() says about it. The reference is + // null in exactly one situation -- nothing is armed, either because nothing ever was or because + // cancelMetricsExpirationTimeout() cleared it when the node came up -- so non-null already + // answers the question this method asks. + // + // Reading cancel()'s answer instead loses the one window the EXPIRED sentinel was added for. + // Netty flips a HashedWheelTimeout to ST_EXPIRED *before* running its task, and the task then + // clears a whole metric set before reaching its compare-and-set, so for that entire interval + // cancel() returns false while the reference still holds the real timeout. A setEndPoint + // landing there would conclude there was nothing to carry and arm nothing, and the task's own + // compare-and-set then fails against the getAndSet above -- so neither side arms anything, and + // the replacement's eagerly re-registered metric set is left with no countdown at all. That is + // verbatim the outcome the paragraph above says the carry-over exists to prevent. + if (pending != EXPIRED) { + pending.cancel(); + } + try { + startMetricsExpirationTimeout(); + } catch (RuntimeException e) { + LOG.debug("Could not re-arm the metrics expiration timeout, skipping it", e); + } + } + protected Timeout newTimeout() { return context .getNettyOptions() @@ -174,9 +287,57 @@ protected Timeout newTimeout() { .newTimeout( t -> { clearMetrics(); - cancelMetricsExpirationTimeout(); + // Conditional on this timeout still being the current one, which is what makes the + // hand-over a single transition. Netty marks a timeout ST_EXPIRED before invoking its + // task, so a concurrent cancel() -- from the node coming back up, or from + // adoptExpirationFrom claiming the countdown -- can already be failing while this + // task runs; the compare-and-set is how the two agree on which of them won. Losing + // means the reference was cleared or re-armed underneath, and then the expiry is not + // this object's to report. Note that neither of those callers decides anything from + // that failing cancel(): it cannot distinguish "already cancelled" from "expiring as + // we speak", and both of them go by the reference instead. + // + // Not routed through cancelMetricsExpirationTimeout() any more, even though + // MicrometerNodeMetricUpdater and MicroProfileNodeMetricUpdater override it: that + // method means "the node is back, drop the countdown", which is the opposite of what + // happened here, and calling it is what reopened the window, by clearing the flag the + // next statement then had to set. Both overrides are pure super-delegation today, so + // nothing observable moves. + metricsExpirationTimeoutRef.compareAndSet(t, EXPIRED); }, expireAfter.toNanos(), TimeUnit.NANOSECONDS); } + + /** + * The {@link #EXPIRED} marker. Never handed to a timer and never asked to do anything: it only + * has to be distinguishable from a real {@link Timeout} by reference. + */ + private static final class ExpiredSentinel implements Timeout { + + @Override + public Timer timer() { + throw new UnsupportedOperationException("Not a real timeout"); + } + + @Override + public TimerTask task() { + throw new UnsupportedOperationException("Not a real timeout"); + } + + @Override + public boolean isExpired() { + return true; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java index 93d003f0a03..af84782c989 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java @@ -19,4 +19,23 @@ import com.datastax.oss.driver.api.core.metrics.NodeMetric; -public interface NodeMetricUpdater extends MetricUpdater {} +public interface NodeMetricUpdater extends MetricUpdater { + + /** + * Takes over the metrics-expiration countdown from the updater this one is replacing, when a node + * rebuilds its updater after its endpoint changed. + * + *

Without this the countdown is simply lost. It is armed and cancelled through {@code + * node.getMetricUpdater()} -- by the metrics factories, on node state events -- so once a node + * has swapped in a replacement, the cancel that a later UP event triggers reaches the new updater + * and finds nothing, while the old updater's timer is still pending on an object nothing else + * refers to. Both halves of that are wrong: the replacement never expires, because a node that is + * already down will not produce another DOWN event to arm it, and the orphan eventually fires + * {@link #clearMetrics()} on names it recomputes from whatever endpoint the node holds by then. + * + *

Implementations that do not expire metrics can ignore this. + */ + default void adoptExpirationFrom(NodeMetricUpdater previous) { + // nothing to hand over + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/pool/ChannelPool.java b/core/src/main/java/com/datastax/oss/driver/internal/core/pool/ChannelPool.java index c9bc5df2f85..80ab258dcc2 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/pool/ChannelPool.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/pool/ChannelPool.java @@ -27,7 +27,6 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.InvalidKeyspaceException; import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; -import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.connection.ReconnectionPolicy; @@ -553,24 +552,88 @@ private CompletionStage addMissingChannels() { private void handleError( Throwable error, Consumer onFatal, Consumer onKeyspaceError) { + // ChannelFactory.isAuthOnly, not a bare instanceof: one failure no longer means one address. + // A node whose endpoint is a name reports a single failure with the other addresses' failures + // attached as suppressed, and the factory promotes an authentication failure over transport + // ones -- so [refused, refused, auth] would otherwise count as errors.connection.auth alone, + // leaving errors.connection.init at zero while most of the node is unreachable, and tell the + // operator their credentials are wrong. The routing below is unaffected: which failure to act + // on is already decided by ChannelFactory#surfacedFailure. + boolean authOnly = ChannelFactory.isAuthOnly(error); ((DefaultNode) node) .getMetricUpdater() .incrementCounter( - error instanceof AuthenticationException + authOnly ? DefaultNodeMetric.AUTHENTICATION_ERRORS : DefaultNodeMetric.CONNECTION_INIT_ERRORS, null); + if (!authOnly && ChannelFactory.mentionsAuthentication(error)) { + // Mixed [refused, refused, auth]: both things really did happen, so both are counted. + // Routing the mixed case to errors.connection.init alone would be the opposite mistake to + // the one above -- this method is the driver's only writer of errors.connection.auth, so + // for any node whose endpoint is a name (SNI/cloud proxy, a client route, or a translator + // with resolve-addresses = false) that metric could never leave zero, however wrong the + // credentials are. An operator watching it would see nothing while every connect failed on + // authentication. Counting both keeps errors.connection.init honest about the unreachable + // addresses without making the auth signal unobservable. + // + // mentionsAuthentication, not `error instanceof AuthenticationException`: the surfaced + // failure is chosen by ChannelFactory#surfacedFailure, which ranks an invalid keyspace -- + // and a node-wide failure -- above an authentication one. So [auth, no-such-keyspace] + // arrives as the keyspace error with the auth failure suppressed, and a test on the type + // of what arrived would leave errors.connection.auth at zero in exactly the case this + // branch exists for. + ((DefaultNode) node) + .getMetricUpdater() + .incrementCounter(DefaultNodeMetric.AUTHENTICATION_ERRORS, null); + } + // What the operator is told about credentials, decided *before* and separately from what the + // driver does about the failure. The two used to be one if/else chain, which meant the fatal + // types -- tested first, because they end the pool's loop -- swallowed the auth diagnosis + // whole: a node whose addresses went [auth, unsupported-protocol-version] surfaces the + // version rejection (node-wide, so ChannelFactory#surfacedFailure promotes it), got forced + // down permanently, and logged nothing at any level about the rejected login. The counter + // above had already moved, so errors.connection.auth climbed while every message named the + // protocol version, and the credential rejection was reachable only by reading + // getSuppressed() + // off the logged throwable. + boolean warnedAboutAuth = true; + if (authOnly) { + // Always warn because this is most likely something the operator needs to fix. + // Keep going to reconnect if it can be fixed without bouncing the client. + Loggers.warnWithException(LOG, "[{}] Authentication error", logPrefix, error); + } else if (ChannelFactory.mentionsAuthentication(error)) { + // Authentication on some addresses, something else on the others: the credentials are not + // the whole story, so the message says so -- but this still warns unconditionally, exactly + // like the auth-only branch above. advanced.connection.warn-on-init-error exists to mute + // the noise of nodes that cannot be reached; it was never a switch for "your credentials + // are wrong", and before multi-address support every AuthenticationException warned here + // regardless of it. Gating the mixed case on it would mean a name whose records fail + // [refused, refused, auth] logs at DEBUG, so the one part of the failure the operator can + // actually fix is the part they never see. + // + // mentionsAuthentication for the same reason the counter above uses it: surfacedFailure + // ranks an invalid keyspace, and a node-wide failure, above an authentication one, so the + // rejected login often arrives suppressed rather than as the throwable itself. + Loggers.warnWithException( + LOG, + "[{}] Error while opening new channel (authentication failed on some of the node's" + + " addresses, other addresses failed for other reasons)", + logPrefix, + error); + } else { + warnedAboutAuth = false; + } + + // And what to do about it. Which failure of the set is acted on was already decided by + // ChannelFactory#surfacedFailure; this only routes it. if (error instanceof ClusterNameMismatchException || error instanceof UnsupportedProtocolVersionException) { // This will likely be thrown by all channels, but finish the loop cleanly onFatal.accept(error); - } else if (error instanceof AuthenticationException) { - // Always warn because this is most likely something the operator needs to fix. - // Keep going to reconnect if it can be fixed without bouncing the client. - Loggers.warnWithException(LOG, "[{}] Authentication error", logPrefix, error); } else if (error instanceof InvalidKeyspaceException) { onKeyspaceError.accept(null); - } else { + } else if (!warnedAboutAuth) { if (config.getDefaultProfile().getBoolean(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR)) { Loggers.warnWithException(LOG, "[{}] Error while opening new channel", logPrefix, error); } else { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java index 8905edb9192..d4f2108ca7d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java @@ -18,6 +18,9 @@ package com.datastax.oss.driver.internal.core.util; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; +import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -56,4 +59,170 @@ public static Set extract(String address, boolean resolve) { return result; } } + + /** + * Whether {@code address} denotes a host name, as opposed to an IP address written out in + * literal form. + * + *

The distinction matters wherever a name is treated as something that can be resolved — and + * re-resolved — while a literal is taken as the final answer. Both forms can appear resolved or + * unresolved, so neither {@link InetSocketAddress#isUnresolved()} nor the presence of an {@link + * InetAddress} tells them apart. + * + *

Performs no lookup of any kind. + * + *

The answer is only stable for an {@linkplain InetSocketAddress#isUnresolved() unresolved} + * address. A resolved one has no host string of its own: {@code getHostString()} renders the + * {@link InetAddress}'s cached {@code hostName} field, which is empty until the first time + * anything calls {@code getHostName()} on that instance and holds a reverse-DNS name afterwards — + * and {@code DefaultSslEngineFactory} calls it while building an engine, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. So a resolved address built + * over an IP literal answers {@code false} before the first TLS handshake to that node and {@code + * true} after it, for the very same instance. Callers that must not flip with it either restrict + * themselves to unresolved addresses (as {@code ChannelFactory#reattachHostname} does) or strip + * the label first (see {@link #stripHostName}). + * + *

On the unresolved branch, a scoped IPv6 literal (say {@code fe80::1%eth0}) and a + * bracketed one ({@code [2001:db8::5]}, the spelling {@link #extract} preserves) are both + * correctly reported as literals. Callers that go on to parse the string must therefore be ready + * for a zone and for brackets — {@link + * com.datastax.oss.driver.shaded.guava.common.net.InetAddresses#forString} rejects the bracketed + * form outright, and resolves a zone against the local interfaces, throwing {@link + * IllegalArgumentException} when no interface matches (see {@code + * ChannelFactory#reattachHostname}, which strips both before parsing). + */ + public static boolean carriesName(InetSocketAddress address) { + String hostString = address.getHostString(); + if (hostString == null) { + return false; + } + // A resolved address is compared against the literal its own bytes produce, which is cheaper + // and + // stricter than parsing; an unresolved one has no bytes, so its string has to be parsed. + InetAddress ip = address.getAddress(); + return ip != null ? !hostString.equals(ip.getHostAddress()) : !isLiteral(hostString); + } + + /** + * Whether an unresolved address's host string is an IP address in literal form. + * + *

Two spellings count. {@code InetAddresses#isInetAddress} accepts the bare form, zone + * included ({@code 2001:db8::5}, {@code fe80::1%eth0}); only {@code + * InetAddresses#isUriInetAddress} accepts the bracketed URI form ({@code [2001:db8::5]}). + * + *

The bracketed form is not hypothetical. {@link #extract} splits a contact point on its + * last colon and keeps whatever precedes it verbatim, so {@code [2001:db8::5]:9042} yields + * the host string {@code [2001:db8::5]}, and {@code InetAddress.getAllByName} accepts that + * spelling — the configuration works end to end. Testing the bare form alone would report it as a + * host name, which costs on both sides: {@code DefaultEndPoint#equals} would fire the mixed + * unresolved/resolved warning and burn its once-per-JVM canary on a message whose stated hazards + * are all false for a literal, and {@code ChannelFactory#reattachHostname} would take the + * name-wins branch and relabel even a resolver-redirected candidate, bypassing the byte-equality + * guard the literal branch exists for. + */ + private static boolean isLiteral(String hostString) { + return InetAddresses.isInetAddress(hostString) || InetAddresses.isUriInetAddress(hostString); + } + + /** + * Parses {@code hostString} as an IP address literal -- the same two spellings {@link #isLiteral} + * recognises -- or returns {@code null} if it is not one. Performs no lookup. + * + *

Here, beside the recognition, because the two have to accept the same strings and nothing + * but proximity makes them: a caller that recognises a literal with {@link #carriesName} and then + * parses it with a grammar of its own has two grammars to keep in agreement, and only a comment + * saying so. + * + *

Neither Guava predicate has a matching parser. {@code InetAddresses#forString} rejects the + * bracketed URI form outright, and it resolves an IPv6 zone against the local interfaces, + * throwing when the zone names none -- it rejects even {@code fe80::1%lo} on a host that has an + * {@code lo} interface. So the brackets come off and the zone is split away before the parse. + * + *

Brackets first, then the zone: {@link #extract} splits a contact point on its last + * colon and keeps them, so {@code [fe80::1%eth0]:9042} arrives here as {@code [fe80::1%eth0]} -- + * splitting on {@code '%'} before unwrapping would leave the closing bracket inside the zone and + * the opening one inside the literal, and neither part would parse. + * + *

The zone is dropped rather than resolved, so the result carries the literal's bytes + * and nothing else. A caller comparing it is therefore scope-blind -- which {@link + * InetAddress#equals} is anyway -- and one that needs the zone should keep the original string. + */ + @Nullable + public static InetAddress parseLiteral(String hostString) { + String bare = hostString; + if (bare.length() > 2 && bare.charAt(0) == '[' && bare.charAt(bare.length() - 1) == ']') { + bare = bare.substring(1, bare.length() - 1); + } + int zoneSeparator = bare.indexOf('%'); + String literalPart = zoneSeparator < 0 ? bare : bare.substring(0, zoneSeparator); + try { + return InetAddresses.forString(literalPart); + } catch (IllegalArgumentException notALiteral) { + return null; + } + } + + /** + * Returns a copy of {@code ip} labelled with {@code hostName}, or with no label at all when + * {@code hostName} is {@code null}, preserving an IPv6 zone if there is one. + * + *

{@link InetAddress#getByAddress(String, byte[])} cannot carry a zone, and dropping one would + * change where the address actually points — a link-local address is only meaningful together + * with its zone. {@link Inet6Address#getByAddress(String, byte[], int)} carries the zone as its + * numeric id, which is what the connect itself goes on. + * + *

That overload is used only for an address that really has a zone. {@code + * Inet6AddressHolder.init} treats any {@code scope_id >= 0} as zone-present, so handing it the + * {@code 0} an unscoped address reports produces a spurious {@code %0} suffix — verified on JDK + * 11.0.30: {@code Inet6Address.getByAddress("db.example.com", bytes, 0).getHostAddress()} is + * {@code "2001:db8:0:0:0:0:0:5%0"}, while the two-arg overload yields the clean form. That suffix + * would reach node metric tags through an endpoint's {@code toString()}, and would break {@link + * #carriesName}'s resolved branch, which needs the host string and the literal to compare equal. + * + *

The sibling overload taking a {@link java.net.NetworkInterface} is deliberately not used: it + * re-derives the numeric zone by searching that interface for an address of the same local type, + * and throws {@code UnknownHostException("no scope_id found")} when it finds none — so it can + * fail for an address that was legitimately built from an interface in the first place. All that + * is lost by going numeric is the interface name, which surfaces in {@code toString()} and + * nowhere else. + * + *

Performs no lookup. + */ + public static InetAddress withHostName(@Nullable String hostName, InetAddress ip) + throws UnknownHostException { + if (ip instanceof Inet6Address) { + int scopeId = ((Inet6Address) ip).getScopeId(); + if (scopeId != 0) { + return Inet6Address.getByAddress(hostName, ip.getAddress(), scopeId); + } + } + return InetAddress.getByAddress(hostName, ip.getAddress()); + } + + /** + * Returns {@code address} with its host-name label removed, so that {@code getHostString()} + * reports the IP literal and cannot start reporting something else later, or {@code null} if + * {@code address} carries no {@link InetAddress} to strip. + * + *

Anything deriving a durable identity from a resolved address's host string has to do + * this first: that string renders a mutable field on the shared {@code InetAddress} (see {@link + * #carriesName}), so an identity keyed off it moves the first time the node is connected to over + * TLS. Stripping makes the identity a function of the address bytes alone. + * + *

Performs no lookup. + */ + @Nullable + public static InetSocketAddress stripHostName(InetSocketAddress address) { + InetAddress ip = address.getAddress(); + if (ip == null) { + return null; + } + try { + return new InetSocketAddress(withHostName(null, ip), address.getPort()); + } catch (UnknownHostException impossible) { + // getByAddress only rejects illegal byte lengths, and these bytes come from a real + // InetAddress. + return null; + } + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/collection/QueryPlan.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/collection/QueryPlan.java index a5ca3443efc..a72bc1f8400 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/collection/QueryPlan.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/collection/QueryPlan.java @@ -39,8 +39,12 @@ * methods throw. * *

Both {@link #size()} and {@link #iterator()} are supported and never throw, even if called - * concurrently. These methods are implemented for reporting purposes only, the driver itself does - * not use them. + * concurrently. They exist mainly for reporting, and the driver's request path does not use them -- + * but that guarantee is load-bearing rather than merely documented: {@code + * LoadBalancingPolicyWrapper#newControlReconnectionQueryPlan} asks {@code isEmpty()} (which is + * {@code size() == 0}) of the plan a policy returned, to decide whether a reconnection round has + * any live node to try before falling back to the contact points. A custom implementation that + * throws from either method breaks the control connection's reconnection, not just a report. * *

All built-in {@link QueryPlan} implementations can be safely reused for custom load balancing * policies; if you plan to do so, study the source code of {@link diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 590784b70c5..5c2fcf682ae 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -541,6 +541,35 @@ datastax-java-driver { # Overridable in a profile: no max-orphan-requests = 256 + # The maximum number of addresses a single connection attempt will try, when the endpoint it + # connects to is a DNS name that resolves to several addresses. + # + # Each address tried is a full TCP connect plus protocol handshake -- with wrong credentials, + # that includes a rejected login. This cap bounds what one attempt can cost in time and in + # login attempts. + # + # Whether the addresses are shuffled first depends on the endpoint. For a contact point, and for + # a node reached through the Cloud SNI proxy or a cloud private-endpoint client route, every + # address is another way in to the same place, so the order is shuffled on every attempt and + # addresses beyond the cap are not lost: successive attempts (e.g. reconnection rounds) sample + # different subsets. For a node whose address came from an `AddressTranslator` that returned a + # name (`SubnetAddressTranslator` does, under `resolve-addresses = false`), that name is not + # known to cover only that one node, so the resolver's order is kept instead -- a pool then + # converges on one address and the rest serve as fallback, but the cap is a hard limit and + # records beyond it are never reached. + # + # Sampling across attempts also needs there to be more than one attempt. At session + # initialization there is exactly one, unless `advanced.reconnect-on-init` is enabled, so a name + # with more records than the cap can fail `build()` while a healthy address goes untried. + # + # Setting this to 1 restores pre-multi-address behavior: one address tried per attempt. + # + # Required: yes + # Modifiable at runtime: yes, the new value will be used for connections created after the + # change. + # Overridable in a profile: no + max-candidate-addresses = 5 + # Whether to log non-fatal errors when the driver tries to open a new connection. # # This error as recoverable, as the driver will try to reconnect according to the reconnection @@ -1233,27 +1262,25 @@ datastax-java-driver { } - # Whether to resolve the addresses passed to `basic.contact-points`. + # DEPRECATED: this option no longer has any effect and will be removed in a future release. # - # If this is true, addresses are created with `InetSocketAddress(String, int)`: the host name will - # be resolved the first time, and the driver will use the resolved IP address for all subsequent - # connection attempts. + # Contact points given here are now always kept as unresolved hostnames and expanded to all of + # their DNS-mapped IPs lazily at connection time. This means the driver tries every IP a hostname + # resolves to, and re-resolves the hostname on each new connection so DNS changes are picked up + # automatically. Previously this option selected between resolving a contact-point hostname once + # (true) and re-resolving it on every connection (false); that distinction no longer applies. # - # If this is false, addresses are created with `InetSocketAddress.createUnresolved()`: the host - # name will be resolved again every time the driver opens a new connection. This is useful for - # containerized environments where DNS records are more likely to change over time (note that the - # JVM and OS have their own DNS caching mechanisms, so you might need additional configuration - # beyond the driver). + # The lookup goes through Netty's configured AddressResolverGroup -- the same resolver an + # unresolved address would have reached had it been passed straight to Bootstrap.connect() -- so a + # custom resolver installed via NettyOptions.afterBootstrapInitialized() still applies. With + # Netty's default (JDK) resolver the lookup blocks the I/O event loop it runs on; install + # DnsAddressResolverGroup if you need it to be non-blocking. # - # This option only applies to the contact points specified in the configuration. It has no effect - # on: - # - programmatic contact points passed to SessionBuilder.addContactPoints: these addresses are - # built outside of the driver, so it is your responsibility to provide unresolved instances. - # - dynamically discovered peers: the driver relies on Cassandra system tables, which expose raw - # IP addresses. Use a custom address translator to convert them to unresolved addresses (if - # you're in a containerized environment, you probably already need address translation anyway). + # This option only ever applied to the contact points specified in the configuration -- never to + # programmatic contact points passed to SessionBuilder.addContactPoints, nor to dynamically + # discovered peers. # - # Required: no (defaults to false) + # Required: no # Modifiable at runtime: no # Overridable in a profile: no advanced.resolve-contact-points = false @@ -2347,14 +2374,70 @@ datastax-java-driver { } reconnection { - # Whether to forcibly add original contact points held by MetadataManager to the reconnection plan, - # in case there is no live nodes available according to LBP. - # Experimental. + # Whether to append the original contact points held by MetadataManager to the reconnection + # plan, after the live nodes reported by the load balancing policy. + # + # This is also the driver's DNS re-resolution path. Contact points are kept as unresolved + # hostnames and expanded to their current DNS IPs at connection time, through Netty's + # configured resolver. Metadata nodes, in contrast, store an already-resolved endpoint that + # is never re-resolved, so once DNS records change they would otherwise become stale. Keeping + # this enabled lets control-connection reconnects re-resolve the original hostnames and pick up + # the new IPs once the live-node plan is exhausted. + # + # Note the cost: the contact points are appended without being compared against the live-node + # plan, because at plan time they are still hostnames and the live nodes are already-resolved + # IPs. When DNS has not changed they therefore expand to addresses the plan just failed on, so + # a reconnection round that exhausts the live nodes retries them a second time. + # + # How much that adds depends on how many addresses each contact point resolves to: a live node + # is one address, while a contact point is expanded to up to + # `advanced.connection.max-candidate-addresses` of them (5 by default). So three contact points + # can append up to 15 attempts to a round, not 3. Each attempt costs a connect -- up to + # `advanced.connection.connect-timeout` -- and, if the address accepts the connection but then + # stalls, the init handshake on top, whose steps each arm their own + # `advanced.connection.init-query-timeout`. On top of that, every appended contact point is an + # unidentified node, so the control connection identifies it inside the candidate loop: one + # `SELECT * FROM system.local` per candidate address, bounded by + # `advanced.control-connection.timeout`. That read is unprojected, and it also discards the + # column projection learned from the previous control node, so the first query after a + # reconnect through a contact point is a `SELECT *` as well. The attempts are serial, and the + # control connection stays down for the whole round. Lowering `max-candidate-addresses` bounds + # that instead. + # + # Setting this to false switches DNS re-resolution off for the ordinary case, where every + # metadata node holds an already-resolved address and so does the node the control connection + # is on -- registered under the address it reached rather than under the contact point it was + # reached through. So set it to false only when the contact points are IP literals, or when + # their records never change and you would rather keep reconnection rounds short. + # + # Two kinds of deployment are exceptions, and for them this option is close to a no-op: + # + # - An AddressTranslator that returns a hostname, as SubnetAddressTranslator does under + # `resolve-addresses = false`, and as FixedHostNameAddressTranslator and + # Ec2MultiRegionAddressTranslator now do. Those endpoints re-expand on every attempt + # whatever this is set to. + # - A Cloud (SNI) session. Every node's endpoint there is built as an SniEndPoint, which + # keeps the proxy address unresolved, so it too re-expands per attempt. And the append + # itself is skipped: the driver does not add contact points behind a topology monitor that + # re-resolves node addresses on its own, unless the live-node plan came back empty. So the + # extra attempts costed above are not being paid there either, and turning this off + # removes only that empty-plan fallback -- which is the one case a reconnection cannot + # recover from without the contact points. + # + # A cloud private-endpoint (client routes) session is NOT one of those exceptions, despite + # also going through a proxy. Its endpoints are client-route endpoints, and one re-expands a + # route hostname per attempt only while a route exists for that node; for a node with no + # route it hands out a static, already-resolved fallback address that is never re-resolved. + # The driver reports such a session as re-resolving its own addresses only while *every* + # known node has a live route, so in a partially routed cluster the contact points are + # appended as usual -- the costs above apply -- and this option is the only DNS + # re-resolution any route-less node gets. Setting it to false there is not the no-op it is + # for Cloud. # # Required: yes # Modifiable at runtime: yes, the new value will be used for checks issued after the change. # Overridable in a profile: no - fallback-to-original-contact-points = false + fallback-to-original-contact-points = true } } diff --git a/core/src/test/java/com/datastax/dse/driver/internal/core/insights/InsightsClientTest.java b/core/src/test/java/com/datastax/dse/driver/internal/core/insights/InsightsClientTest.java index 5085432dbec..3a1b690ead8 100644 --- a/core/src/test/java/com/datastax/dse/driver/internal/core/insights/InsightsClientTest.java +++ b/core/src/test/java/com/datastax/dse/driver/internal/core/insights/InsightsClientTest.java @@ -77,6 +77,7 @@ import com.tngtech.java.junit.dataprovider.UseDataProvider; import io.netty.channel.DefaultEventLoop; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.Collections; @@ -168,8 +169,13 @@ public void should_construct_json_event_startup_message() throws IOException { assertThat(insightData.getApplicationName()).isEqualTo("app-name"); assertThat(insightData.getApplicationVersion()).isEqualTo("1.0.0"); assertThat(insightData.isApplicationNameWasGenerated()).isEqualTo(false); + // Keyed on the literal the fixture configured, not on what the loopback's reverse zone calls + // it. This used to read "localhost": the contact point is a *resolved* InetSocketAddress built + // from an IP-literal string, which carries no host name, so grouping on getHostName() reverse + // -resolved it -- on the admin executor, at startup and at every monitor-reporting interval + // after it. See InsightsClient#getResolvedContactPoints. assertThat(insightData.getContactPoints()) - .isEqualTo(ImmutableMap.of("localhost", Collections.singletonList("127.0.0.1:9999"))); + .isEqualTo(ImmutableMap.of("127.0.0.1", Collections.singletonList("127.0.0.1:9999"))); assertThat(insightData.getInitialControlConnection()).isEqualTo("127.0.0.1:10"); assertThat(insightData.getLocalAddress()).isEqualTo("127.0.0.1"); @@ -240,6 +246,30 @@ public void should_group_contact_points_by_host_name() { assertThat(resolvedContactPoints).isEqualTo(expected); } + @Test + public void should_not_reverse_resolve_a_contact_point_that_carries_no_name() throws Exception { + // addContactPoints(new InetSocketAddress(InetAddress.getByAddress(...), port)) yields a + // resolved address with no label, and the driver hands it through unchanged. Keying on + // getHostName() would send that through InetAddress#getHostName() -- a reverse lookup, on the + // admin executor, repeated at every advanced.monitor-reporting interval -- and then name the + // group after whatever the reverse zone said rather than after anything the operator wrote. + // getHostString() answers the same for every case the field is actually about and looks + // nothing up. + // + // Discriminating wherever the loopback has a reverse mapping, which is any machine with an + // ordinary /etc/hosts: getHostName() answers "localhost" there. Where it has none it answers + // the literal too, so this can pass without proving anything but can never fail wrongly. + Set contactPoints = + ImmutableSet.of( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 9042)); + + Map> resolvedContactPoints = + InsightsClient.getResolvedContactPoints(contactPoints); + + assertThat(resolvedContactPoints) + .isEqualTo(ImmutableMap.of("127.0.0.1", ImmutableList.of("127.0.0.1:9042"))); + } + @Test public void should_construct_json_event_status_message() throws IOException { // given @@ -277,6 +307,38 @@ public void should_construct_json_event_status_message() throws IOException { "127.0.0.1:20", new SessionStateForNode(2, 20))); } + @Test + public void should_merge_nodes_that_report_under_the_same_address() throws IOException { + // Behind an SNI proxy or a cloud client route, every node's endpoint resolves to the same proxy + // address, so the map key connectedNodes is built from is not unique per node. Collectors.toMap + // throws IllegalStateException on a duplicate key, which would propagate out of + // createStatusMessage() and abort the status report on every interval for any such deployment + // with more than one node open. + DefaultDriverContext context = mockDefaultDriverContext(); + mockConnectionPoolsBehindOneProxy(context); + InsightsClient insightsClient = + new InsightsClient( + context, + MOCK_TIME_SUPPLIER, + INSIGHTS_CONFIGURATION, + null, + null, + null, + null, + null, + EMPTY_STACK_TRACE); + + // when + String statusMessage = insightsClient.createStatusMessage(); + + // then -- one entry, carrying the totals reached through that address. + Insight insight = + new ObjectMapper() + .readValue(statusMessage, new TypeReference>() {}); + assertThat(insight.getInsightData().getConnectedNodes()) + .isEqualTo(ImmutableMap.of("proxy.example.com:9042", new SessionStateForNode(3, 30))); + } + @Test public void should_schedule_task_with_initial_delay() { // given @@ -510,6 +572,32 @@ private DefaultDriverContext mockDefaultDriverContext() throws UnknownHostExcept return context; } + /** Two nodes whose endpoints resolve to one and the same (unresolved) proxy address. */ + private void mockConnectionPoolsBehindOneProxy(DefaultDriverContext driverContext) { + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + + Node node1 = mock(Node.class); + EndPoint endPoint1 = mock(EndPoint.class); + when(endPoint1.resolve()).thenReturn(proxy); + when(node1.getEndPoint()).thenReturn(endPoint1); + when(node1.getOpenConnections()).thenReturn(1); + ChannelPool channelPool1 = mock(ChannelPool.class); + when(channelPool1.getInFlight()).thenReturn(10); + + Node node2 = mock(Node.class); + EndPoint endPoint2 = mock(EndPoint.class); + when(endPoint2.resolve()).thenReturn(proxy); + when(node2.getEndPoint()).thenReturn(endPoint2); + when(node2.getOpenConnections()).thenReturn(2); + ChannelPool channelPool2 = mock(ChannelPool.class); + when(channelPool2.getInFlight()).thenReturn(20); + + PoolManager poolManager = mock(PoolManager.class); + when(poolManager.getPools()) + .thenReturn(ImmutableMap.of(node1, channelPool1, node2, channelPool2)); + when(driverContext.getPoolManager()).thenReturn(poolManager); + } + private void mockConnectionPools(DefaultDriverContext driverContext) { Node node1 = mock(Node.class); EndPoint endPoint1 = mock(EndPoint.class); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslatorTest.java index 2b871b3e0cc..b6f2d367c8e 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslatorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/Ec2MultiRegionAddressTranslatorTest.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.addresstranslation; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; @@ -56,16 +57,54 @@ public void should_return_same_address_when_exception_encountered() throws Excep } @Test - public void should_return_new_address_when_match_found() throws Exception { - InetSocketAddress expectedAddress = new InetSocketAddress("54.32.55.66", 9042); + public void should_return_same_address_when_the_domain_name_does_not_resolve() throws Exception { + // The third way this translator can fail, and the one the deferred forward lookup nearly took + // away: the PTR record answers, but the name it gives has no A record -- private DNS switched + // off on the VPC, split-horizon DNS that serves the reverse zone only, a stale PTR after an + // instance replacement. Handing that name over unchecked would strand the node for good, since + // the connect layer has nothing to fall back to and every refresh re-derives the same name. + // So the forward lookup still runs here, and its failure means the node keeps the raw + // broadcast address it was already reachable on. + assumeThat(new InetSocketAddress("node1.eu-west-1.example.com", 9042).isUnresolved()) + .as("requires a host whose resolver does not answer for unregistered names") + .isTrue(); InitialDirContext mock = mock(InitialDirContext.class); when(mock.getAttributes("5.2.0.192.in-addr.arpa", new String[] {"PTR"})) - .thenReturn(new BasicAttributes("PTR", expectedAddress.getHostName())); + .thenReturn(new BasicAttributes("PTR", "node1.eu-west-1.example.com")); Ec2MultiRegionAddressTranslator translator = new Ec2MultiRegionAddressTranslator(mock); InetSocketAddress address = new InetSocketAddress("192.0.2.5", 9042); - assertThat(translator.translate(address)).isEqualTo(expectedAddress); + assertThat(translator.translate(address)).isEqualTo(address); + } + + @Test + public void should_not_resolve_a_domain_name_that_would_resolve() { + // The "match found" case, and it has to use a name that really resolves: an unresolvable one + // is answered with the original address (see above), and even without that, `new + // InetSocketAddress(String, int)` leaves it unresolved too, so the two spellings would compare + // equal on host string and port and this could not tell them apart. + assumeThat(new InetSocketAddress("localhost", 9042).isUnresolved()) + .as("requires a host where localhost resolves; where it does not, both spellings agree") + .isFalse(); + + InitialDirContext mock = mock(InitialDirContext.class); + Ec2MultiRegionAddressTranslator translator; + try { + when(mock.getAttributes("5.2.0.192.in-addr.arpa", new String[] {"PTR"})) + .thenReturn(new BasicAttributes("PTR", "localhost")); + translator = new Ec2MultiRegionAddressTranslator(mock); + } catch (NamingException impossible) { + throw new AssertionError(impossible); + } + + // The forward lookup belongs to ChannelFactory, which expands the name to every address it + // maps to. Doing it here keeps one and the rest are never tried, because the resolver reports + // an already-resolved address as nothing to do. + InetSocketAddress translated = translator.translate(new InetSocketAddress("192.0.2.5", 9042)); + assertThat(translated.isUnresolved()).isTrue(); + assertThat(translated.getHostString()).isEqualTo("localhost"); + assertThat(translated.getPort()).isEqualTo(9042); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java index da9c40f033c..ea5d628bf4a 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslatorTest.java @@ -19,6 +19,7 @@ import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,4 +46,31 @@ public void should_translate_address() { assertThat(translator.translate(address)) .isEqualTo(InetSocketAddress.createUnresolved("myaddress", 6061)); } + + /** + * The sibling above cannot fail: {@code myaddress} does not resolve, so {@code new + * InetSocketAddress(String, int)} leaves it unresolved too and the two spellings compare equal. A + * name that does resolve is what tells them apart -- and telling them apart is the + * point, because a resolved address is never expanded to the proxy's other addresses. + */ + @Test + public void should_not_resolve_a_hostname_that_would_resolve() { + assumeThat(new InetSocketAddress("localhost", 6061).isUnresolved()) + .as("requires a host where localhost resolves; where it does not, both spellings agree") + .isFalse(); + + DriverExecutionProfile defaultProfile = mock(DriverExecutionProfile.class); + when(defaultProfile.getString(ADDRESS_TRANSLATOR_ADVERTISED_HOSTNAME)).thenReturn("localhost"); + DefaultDriverContext defaultDriverContext = + MockedDriverContextFactory.defaultDriverContext(Optional.of(defaultProfile)); + + FixedHostNameAddressTranslator translator = + new FixedHostNameAddressTranslator(defaultDriverContext); + + InetSocketAddress translated = translator.translate(new InetSocketAddress("192.0.2.5", 6061)); + + assertThat(translated.isUnresolved()).isTrue(); + assertThat(translated.getHostString()).isEqualTo("localhost"); + assertThat(translated.getPort()).isEqualTo(6061); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java new file mode 100644 index 00000000000..dbe60ae4432 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.context.NettyOptions; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelInboundHandlerAdapter; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies the {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} contract: the hook runs on + * a handler-less bootstrap, and a handler it installs is replaced by the driver's own. + */ +public class ChannelFactoryBootstrapHookTest extends ChannelFactoryTestBase { + + @Test + public void should_replace_handler_installed_by_bootstrap_hook() { + // Given – a hook that (incorrectly) installs its own channel handler. The driver sets its own + // handler on each per-attempt copy afterwards, logging a one-time warning; if the dummy + // handler below survived instead, the protocol handshake would never happen and this connect + // would fail on the init timeout. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.handler(new ChannelInboundHandlerAdapter()); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java new file mode 100644 index 00000000000..1013510e4e2 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java @@ -0,0 +1,880 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThat; +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverConfig; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.ProtocolConstants; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Register; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Error; +import com.datastax.oss.protocol.internal.response.Ready; +import io.netty.channel.local.LocalAddress; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.EventExecutorGroup; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +/** + * Verifies the two steps {@link ChannelFactory} runs between protocol initialization and the + * completion of a candidate attempt: the caller's {@link ConnectHook}, and the REGISTER request + * that moved out of the init handshake so that a channel the hook is about to reject never + * registers for events. + */ +public class ChannelFactoryConnectHookTest extends ChannelFactoryTestBase { + + private static final Duration HOOK_TIMEOUT = Duration.ofSeconds(5); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + /** A local address that no server is bound to: connecting to it fails immediately. */ + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryConnectHookTest.class.getSimpleName() + "-unreachable"); + + private void givenNegotiableProtocol() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + } + + /** + * Drives one candidate's handshake to successful completion. Only the factory's first channel + * sends OPTIONS, so later candidates start straight at STARTUP. + */ + private void completeInit() { + Frame requestFrame = readOutboundFrame(); + if (requestFrame.message instanceof Options) { + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + } + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("mockClusterName")); + } + + private static DriverChannelOptions optionsWithHook(ConnectHook hook) { + return DriverChannelOptions.builder().withConnectHook(hook, HOOK_TIMEOUT).build(); + } + + private static DriverChannelOptions optionsWithHookAndEvents(ConnectHook hook) { + return DriverChannelOptions.builder() + .withConnectHook(hook, HOOK_TIMEOUT) + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + } + + @Test + public void should_complete_candidate_only_after_hook_accepts() { + // Given — a hook whose completion the test controls. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + List vettedChannels = new CopyOnWriteArrayList<>(); + ConnectHook hook = + channel -> { + vettedChannels.add(channel); + return gate; + }; + + // When — init completes but the hook has not answered yet. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt is not successful until the hook says so. (The hook runs on the event + // loop after init succeeds, so wait for the invocation before asserting on the future.) + await().atMost(java.time.Duration.ofSeconds(2)).until(() -> vettedChannels.size() == 1); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + + // When + gate.complete(null); + + // Then — the vetted channel is the one handed to the caller. + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel).isSameAs(vettedChannels.get(0))); + } + + @Test + public void should_try_next_address_when_hook_rejects_candidate() { + // Given — a name expanding to two addresses of the live server, and a hook that rejects the + // first candidate and accepts the second: the caller's acceptance criteria are per address, + // and a rejection must not write off the endpoint. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + AtomicInteger invocations = new AtomicInteger(); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + if (invocations.incrementAndGet() == 1) { + result.completeExceptionally(new IllegalStateException("not this one")); + } else { + result.complete(null); + } + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + optionsWithHook(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + completeInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + assertThat(invocations.get()).isEqualTo(2); + } + + @Test + public void should_fail_connect_when_hook_rejects_the_last_candidate() { + // Given — a single address, so the rejection has nowhere to advance to. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException rejection = new IllegalStateException("cannot identify itself"); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + result.completeExceptionally(rejection); + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the rejection's cause is preserved for diagnosis. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(rejection); + }); + } + + @Test + public void should_treat_synchronous_hook_throw_as_rejection() { + // A hook is a caller-supplied callback running inside a Netty listener: a leaked throwable + // would otherwise leave the attempt hanging forever. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException thrown = new IllegalStateException("hook blew up"); + ConnectHook hook = + channel -> { + throw thrown; + }; + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(thrown); + }); + } + + @Test + public void should_reject_candidate_when_hook_times_out() { + // Given — a hook whose stage never completes; only the driver can bound that. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = channel -> new CompletableFuture<>(); + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ofMillis(100)).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("timed out")); + } + + @Test + public void should_arm_the_hook_timeout_on_neither_thread_the_connect_uses() { + // The wedge the backstop exists for, and the only one it can catch. A hook that hands back a + // stage and never completes it is bounded by any timer at all; a hook that *blocks* is not, and + // blocking is exactly what TopologyMonitor#getChannelNodeInfo's contract has to ask + // implementations not to do, because nothing enforces it. + // + // So it must not be armed on either thread this connect already leans on. Not the channel's + // event loop: the hook runs there -- finishCandidate is reached from a channel-promise + // listener, which Netty notifies on it -- so a task armed on that loop is queued behind the + // very block it was meant to interrupt. And not the admin group either: #connectToAddress + // dispatches the blocking shard-aware port scan there, once per candidate address, on a + // two-thread group one of whose threads is the control connection's own executor. + // + // Which leaves the driver's timer, and where the task goes is the whole property -- so that is + // what this asserts, from both ends. Blocking a loop and racing the deadline instead was the + // first version of this test, and it failed on a slow CI runner rather than on a regression: + // the assertion budget is wall-clock, and a hook blocking a shared runner's core is the last + // thing to add to that. The timeout here is long enough that it never fires. + givenNegotiableProtocol(); + Timer recordingTimer = mock(Timer.class); + when(recordingTimer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenAnswer( + invocation -> + timer.newTimeout( + invocation.getArgument(0), + (long) invocation.getArgument(1), + invocation.getArgument(2))); + when(nettyOptions.getTimer()).thenReturn(recordingTimer); + EventExecutor recordingExecutor = mock(EventExecutor.class); + EventExecutorGroup recordingGroup = mock(EventExecutorGroup.class); + when(recordingGroup.next()).thenReturn(recordingExecutor); + when(nettyOptions.adminEventExecutorGroup()).thenReturn(recordingGroup); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ofMinutes(5)).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then -- on the timer, not on the admin group, and the attempt is otherwise untouched. + verify(recordingTimer, timeout(5000)) + .newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + verify(recordingExecutor, never()) + .schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + gate.complete(null); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_arm_the_hook_timeout_before_calling_the_hook() { + // Where the backstop is armed only matters if it is armed at all, and a hook that blocks + // inside onConnect never lets the arming statement run: the call does not return, so no timer + // exists on any thread and the wedge the previous test describes is bounded by nothing. Which + // thread the task lands on is only half the property; this is the other half. + // + // Asserted as an order rather than by blocking a loop and waiting for a deadline. The version + // of this test that blocked a Netty loop for four seconds raced a wall-clock assertion budget + // and failed on a slow CI runner instead of on a regression, so it was removed; the ordering + // is the actual invariant and needs no clock at all. + givenNegotiableProtocol(); + AtomicBoolean armed = new AtomicBoolean(); + AtomicBoolean armedBeforeTheHookRan = new AtomicBoolean(); + Timer recordingTimer = mock(Timer.class); + when(recordingTimer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenAnswer( + invocation -> { + armed.set(true); + return timer.newTimeout( + invocation.getArgument(0), + (long) invocation.getArgument(1), + invocation.getArgument(2)); + }); + when(nettyOptions.getTimer()).thenReturn(recordingTimer); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = + channel -> { + armedBeforeTheHookRan.set(armed.get()); + return CompletableFuture.completedFuture(null); + }; + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ofMinutes(5)).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then the hook saw a timeout already armed, and accepting still works. + assertThatStage(channelFuture).isSuccess(); + assertThat(armedBeforeTheHookRan).isTrue(); + } + + @Test + public void should_register_for_events_only_after_hook_accepts() { + // Given — events requested and a gated hook. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHookAndEvents(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + // The hook has not accepted yet: were REGISTER part of init, it would already be on the wire. + assertThat(tryReadOutboundFrame(200)).isNull(); + gate.complete(null); + + // Then — REGISTER goes out only now, and the attempt completes once it is acknowledged. + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + assertThat(((Register) registerFrame.message).eventTypes).containsExactly("foo", "bar"); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_fail_candidate_when_the_step_after_the_hook_throws() + throws InterruptedException { + // Given — events requested, and a registration step that throws once the hook has accepted. + // registerForEvents() runs inside the hook stage's whenComplete callback: nobody consumes the + // stage that callback returns, and the hook timeout that would have failed the candidate has + // just been cancelled, so without a blanket catch there the attempt would hang forever. + // + // The thrower is the event-type list rather than a config read, because the init-query timeout + // is now captured beside the pipeline instead of being read here (see + // ChannelFactory#bootstrapAndConnect). What this pins is the catch, not any one way of + // reaching it: the list is consulted first thing in registerForEvents, on the callback's + // thread, and it answers the builder's own emptiness check before that. + givenNegotiableProtocol(); + @SuppressWarnings("unchecked") + List poisonedEventTypes = mock(List.class); + when(poisonedEventTypes.isEmpty()) + .thenReturn(false) + .thenThrow(new IllegalStateException("event types went away")); + ChannelFactory factory = newChannelFactory(); + AtomicReference vettedChannel = new AtomicReference<>(); + ConnectHook hook = + channel -> { + vettedChannel.set(channel); + return CompletableFuture.completedFuture(null); + }; + DriverChannelOptions options = + DriverChannelOptions.builder() + .withConnectHook(hook, Duration.ofMinutes(5)) + .withEvents(poisonedEventTypes, mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt fails instead of hanging, REGISTER never goes out, and the channel it had + // already opened is closed rather than left dangling with nothing holding it. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).hasMessageContaining("event types went away"); + }); + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(vettedChannel.get().closeFuture().await(500, TimeUnit.MILLISECONDS)) + .as("the abandoned candidate's channel should have been closed") + .isTrue(); + } + + @Test + public void should_register_for_events_after_init_when_no_hook_is_set() { + // Given — events but no hook (REGISTER still has to happen even when there is nothing to vet). + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_bound_registration_with_the_timeout_captured_for_the_attempt() { + // advanced.connection.init-query-timeout is documented as taking effect for connections + // created after the change, and every init step honours that by using the value + // ProtocolInitHandler snapshotted when the pipeline was built. REGISTER is sent after init now, + // so reading the option again at that point would hand a connection that already exists a + // value configured after it was created. + // + // At zero that is not just a scope violation. The two request classes disagree about a + // non-positive timeout -- AdminRequestHandler#onWriteComplete arms no timer, while the + // ChannelHandlerRequest the init steps use arms one unconditionally -- so a reload to zero + // landing inside a handshake leaves STARTUP bounded by the old value and REGISTER bounded by + // nothing. Here the server accepts the connection and then never answers REGISTER, which is + // the shape that hangs: no hook is armed either, so nothing else is watching the attempt. + givenNegotiableProtocol(); + when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT)) + .thenReturn(Duration.ofMillis(200)); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When -- the option is disabled once the handshake is done, i.e. inside the connect. + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT)) + .thenReturn(Duration.ZERO); + + // Then REGISTER goes out and is bounded by the 200ms this attempt captured, not by the zero + // that is now configured. No response is written for it. + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("timed out")); + } + + @Test + public void should_try_next_address_when_registration_fails() { + // Given — two addresses; the first candidate's REGISTER is refused. A registration failure is + // a per-candidate failure, exactly as it was when REGISTER was an init step. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then — the loop advances; the second candidate registers successfully. + completeInit(); + registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_treat_a_zero_hook_timeout_as_unbounded() { + // The hook timeout comes from advanced.control-connection.timeout, and every other consumer of + // a + // driver timeout option reads a non-positive duration as "no timeout" (see + // AdminRequestHandler#onWriteComplete). Scheduled anyway, a zero delay fires on the next + // event-loop turn -- before any round trip can complete -- and would abandon every candidate of + // every contact point, so an operator who disabled that timeout could not initialize a session. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ZERO).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the hook is given as long as it needs, and the candidate survives. + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + gate.complete(null); + + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_the_hook_rejects() { + // The protocol version and cluster name used to be latched as soon as the transport connect and + // init handshake succeeded, which was safe while init had the last word on a candidate. It no + // longer does: this hook rejects one, and REGISTER (below) can too. A stale DNS record pointing + // at a foreign cluster would otherwise leave that cluster's name latched here, and every later + // connection -- to any node -- would fail its cluster-name check, which ChannelPool turns into + // an irreversible forced-down node. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = + channel -> { + CompletableFuture rejected = new CompletableFuture<>(); + rejected.completeExceptionally(new IllegalStateException("no host_id")); + return rejected; + }; + + // When — the only candidate is rejected after a fully successful handshake. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("hook")); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_whose_registration_fails() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then + assertThatStage(channelFuture).isFailed(); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_the_timeout_abandoned() { + // The third way a candidate is thrown away, and the one no ordering alone protects against: the + // hook timeout and the hook's own success race. cancel(false) can lose to a timeout task that + // has already started running -- abandonCandidate documents exactly that -- and the candidate + // then walks on through to completeCandidate with its future already failed. + // + // It must not latch on the way past. What decides that is the one-shot settle on the + // candidate's + // future, not the order of the two statements inside completeCandidate: the accepted candidate + // has to latch *before* it publishes, since complete() releases callers on other threads + // synchronously, so "latch only if we then win" is not available. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withConnectHook(channel -> gate, Duration.ofMillis(100)) + .build(); + + // When -- the handshake succeeds, then the timeout fires while the hook is still pending. + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("timed out")); + + // And only now does the hook accept, sending an already-abandoned candidate on to + // completeCandidate -- with no event types requested, straight there and with no REGISTER in + // between, which is what makes this reachable rather than hypothetical. + gate.complete(null); + + // Then -- nothing of that candidate's handshake survives it. + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_fail_the_candidate_when_recording_the_negotiated_state_throws() { + // Winning the settle makes completeCandidate the only call that can still complete the future: + // every blanket catch downstream discharges that duty through abandonCandidate, and that is a + // no-op once the candidate is settled. A throw out of onAccepted would therefore strand the + // attempt -- never completed, channel never closed, Reconnection stuck in ATTEMPT_IN_PROGRESS, + // and nothing left to time it out, since REGISTER is done and the hook timeout is cancelled. + // + // latchNegotiatedState is not throw-free: on the Cloud path it reaches + // TypesafeDriverConfig#overrideDefaults, which re-parses the whole configuration. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + TypesafeDriverConfig typesafeConfig = mock(TypesafeDriverConfig.class); + when(typesafeConfig.getDefaultProfile()).thenReturn(defaultProfile); + doThrow(new IllegalArgumentException("bad reload")) + .when(typesafeConfig) + .overrideDefaults(anyMap()); + when(context.getConfig()).thenReturn(typesafeConfig); + + // A hook and no event types: completeCandidate is then reached from the hook stage's + // whenComplete, whose catch calls abandonCandidate -- the path that would hang. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHook(channel -> CompletableFuture.completedFuture(null)), + NoopNodeMetricUpdater.INSTANCE); + + // The server advertises the Cloud product type, which is what takes latchNegotiatedState into + // the branch that throws. + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame( + requestFrame, TestResponses.supportedResponse("PRODUCT_TYPE", "DATASTAX_APOLLO")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("mockClusterName")); + + // Then -- failed, which is the point: isFailed() waits two seconds and reports a timeout + // rather than blocking, so a hang here shows up as a failure and not as a stuck build. + assertThatStage(channelFuture).isFailed(error -> assertThat(error).isNotNull()); + } + + @Test + public void should_latch_negotiated_state_once_a_candidate_is_accepted() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHook(channel -> CompletableFuture.completedFuture(null)), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture).isSuccess(); + assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V4); + assertThat(factory.getClusterName()).isEqualTo("mockClusterName"); + } + + @Test + public void should_stop_the_candidate_loop_when_the_event_type_rejection_speaks_for_them_all() { + // Given — an identified node with two addresses, and a server that does not support the event + // type being registered for. Every address of an identified node is that same node, so the + // rejection describes all of them: replaying it can only fail the same way while paying a full + // TCP connect plus the STARTUP/AUTH/cluster-name handshake for each. Stopping at the first + // restores what this rejection cost while REGISTER was an init step, which was one failed + // connect per node. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + options, + NoopNodeMetricUpdater.INSTANCE, + /* nodeIsIdentified = */ true); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + // Then — the attempt is already failed, without the second address having been dialled. Compare + // should_try_next_address_when_registration_fails, where the stage is still pending at this + // point because the loop moved on. + // + // The frame check comes first, and it is the assertion that actually binds: surfacedFailure has + // a rung of its own for an UnsupportedEventTypeException, so the type and the message below + // come out whether the loop stopped or not. Draining before the stage assertion also matters on + // regression -- an unread frame parks the server loop in Exchanger#exchange and hangs + // tearDown()'s shutdownGracefully().sync() instead of failing here. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after the event type was rejected") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + assertThat(error.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + + @Test + public void should_try_next_address_when_only_the_first_server_lacks_the_event_type() { + // The same rejection against an unidentified contact point on a plain multi-record name. Those + // records may be distinct servers -- which is what a rolling upgrade looks like from the client + // -- so the one that answered speaks only for itself, and writing the name off would skip the + // upgraded node behind the second record. Only node identity, or an endpoint that says its + // addresses are interchangeable, makes the rejection node-wide. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + // Then — the loop advances; the second address, running newer software, registers successfully. + completeInit(); + registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_report_the_event_type_rejection_over_a_later_address_transport_failure() { + // Having advanced past the rejection (the test above), the loop must still report it. The + // address tried last is arbitrary -- one firewalled record and the failure the caller receives + // is a bare connect timeout, with the only message that says what to do about the deployment + // reachable through getSuppressed(). ClientRoutesTopologyMonitor#init() is the caller, and its + // whole job is to say whether client routes are usable here. + givenNegotiableProtocol(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(SERVER_ADDRESS.resolve(), UNREACHABLE))); + ChannelFactory factory = newChannelFactory(); + // The order matters here, unlike in the test above, so the shuffle is pinned. + factory.random = new KeepResolverOrder(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + // When — the first address answers and rejects the event type, the second is unreachable. + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + // Then — the rejection is what comes out, not the transport failure that happened to be last. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + assertThat(error).hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); + }); + } + + @Test + public void should_translate_client_routes_register_rejection() { + // The one REGISTER rejection with a known cause keeps its clear message, as it had when + // REGISTER was an init step: the caller (ClientRoutesTopologyMonitor.init()) reports it + // instead of silently degrading. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + assertThat(error).hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); + }); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java new file mode 100644 index 00000000000..c1925a9f6dd --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -0,0 +1,1224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Authenticate; +import com.datastax.oss.protocol.internal.response.Ready; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import io.netty.channel.local.LocalAddress; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketAddress; +import java.net.SocketException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.Test; + +/** + * Verifies how {@link ChannelFactory#connect} treats the several addresses a name expands to: they + * are tried in sequence in a shuffled order, at most {@code + * advanced.connection.max-candidate-addresses} of them, and failures are aggregated rather than + * dropped. + * + *

The expansion itself is exercised in {@link ChannelFactoryNettyResolverTest}; here the + * resolver is only the mechanism for producing more than one address from a single endpoint. + */ +public class ChannelFactoryMultiAddressTest extends ChannelFactoryTestBase { + + // Local addresses that no server is bound to: connecting to them fails immediately. + private static final SocketAddress UNREACHABLE_1 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-1"); + private static final SocketAddress UNREACHABLE_2 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-2"); + private static final SocketAddress UNREACHABLE_3 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-3"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_fail_with_suppressed_causes_when_all_addresses_are_unreachable() { + // Given – a name that expands to two dead addresses. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then -- the future fails, and the earlier address's failure is preserved on the last one's + // error rather than being silently dropped. + // + // The count of suppressed entries cannot say that, for the reason the max-candidates test below + // spells out: a single dead candidate already contributes two entangled failures, the transport + // refusal plus the init-write failure PromiseCombiner attaches to it. So getSuppressed() is + // non-empty with one address dialled and nothing carried, and isNotEmpty() would hold with the + // carrying removed outright. The set of addresses named anywhere in the aggregate is what + // reflects what was carried. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(mentionedUnreachableAddresses(e)) + .as("both addresses' failures should be reachable from the surfaced error") + .hasSize(2)); + } + + @Test + public void should_attach_each_earlier_failure_at_most_once() { + // Given – three dead addresses, so there are earlier failures to carry. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – attaching to the error being reported mutates an object the driver does not own, so + // every cause has to appear at most once and the error must never suppress itself. Nothing + // stops two candidates from failing with the same instance -- a pipeline handler that throws a + // stackless singleton, say -- and such an instance would otherwise grow a suppressed entry on + // every connect for as long as the JVM lives. + assertThatStage(channelFuture) + .isFailed( + e -> { + List suppressed = Arrays.asList(e.getSuppressed()); + assertThat(suppressed).isNotEmpty(); + for (int i = 0; i < suppressed.size(); i++) { + assertThat(suppressed.get(i)).isNotSameAs(e); + for (int j = i + 1; j < suppressed.size(); j++) { + assertThat(suppressed.get(i)).isNotSameAs(suppressed.get(j)); + } + } + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_a_contact_point() { + // Given – a name expanding to two addresses, both the same live server, which asks for + // authentication the driver has no provider for. The endpoint is a bare contact point, so the + // driver does not yet know which node -- or even which cluster -- any of these addresses is. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – the loop advances. Authentication completes before the cluster-name check + // (ProtocolInitHandler runs STARTUP -> AUTH_RESPONSE -> GET_CLUSTER_NAME), so a stale record + // pointing at a foreign cluster that wants different credentials fails here rather than at the + // cluster-name mismatch that would have advanced. Writing off the whole name on this error + // would therefore make that rule unreachable in exactly the multi-record case this loop is for. + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message) + .as("the second candidate should have been attempted") + .isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // And – once both are exhausted the failure is still an AuthenticationException, with the first + // address's copy attached rather than dropped. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + @Test + public void should_surface_the_authentication_failure_when_another_address_fails_on_transport() { + // Given – a name expanding to the live server (which asks for authentication the driver has no + // provider for) and a dead address. The shuffled order does not matter: whichever is tried + // first, the pass ends with one authentication failure and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, UNREACHABLE_1))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – even when the transport failure is the *last* error, propagating it would report a + // connect failure for what is really a rejected password: callers branch on the type of what + // they receive (ChannelPool#handleError, ControlConnection's auth-specific warning and its + // errors.connection.auth metric), and with a shuffled multi-record name which address happens + // to be tried last is arbitrary. The classified failure wins, and the transport one is still + // attached. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("an authentication failure must not be demoted by a later transport failure") + .isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the transport failure should still be attached") + .hasSize(1); + assertThat(e.getSuppressed()[0]).isNotInstanceOf(AuthenticationException.class); + }); + } + + @Test + public void should_not_surface_a_cluster_name_mismatch_that_only_one_address_reported() { + // Given – a factory that already knows the cluster name (from a first connection), then a name + // expanding to the live server -- which now answers with a *different* cluster name -- and a + // dead address. Whichever order the shuffle picks, the pass ends with one cluster-name mismatch + // and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + CompletionStage firstChannel = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + assertThatStage(firstChannel).isSuccess(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(SERVER_ADDRESS.resolve(), UNREACHABLE_1))); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // The dead address sends nothing, so this drives the live candidate whichever position it got. + // The protocol version and the product type are known by now, hence no OPTIONS request. + writeInboundFrame(readOutboundFrame(), new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("wrongClusterName")); + + // Then – the mismatch must not be the failure that surfaces, not even as the last error of the + // pass. ChannelPool#handleError turns it into TopologyEvent.forceDown and nothing in the driver + // ever reverses one, while one address of a multi-record name fronting another cluster is a + // stale record rather than a verdict about the node. It is still attached, so nothing is lost. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("a mismatch from a single address must not be promoted over the others") + .isNotInstanceOf(ClusterNameMismatchException.class); + assertThat( + Arrays.stream(e.getSuppressed()) + .anyMatch(s -> s instanceof ClusterNameMismatchException)) + .as("the mismatch should still be attached as a suppressed exception") + .isTrue(); + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_an_identified_node() { + // Given – the same server and the same two addresses, but a node the driver has already + // identified (nodeIsIdentified = true, i.e. its host id was read from system.local/peers). + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + failAuthenticationOnNextCandidate(); + + // Then – the loop advances, exactly as it does for a contact point: no single address's + // failure writes off the endpoint, and the candidate cap -- not a node-wide classification -- + // is what bounds the cost of genuinely wrong credentials. + failAuthenticationOnNextCandidate(); + + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + /** Drives one candidate's handshake as far as the server's authentication challenge. */ + private void failAuthenticationOnNextCandidate() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + } + + @Test + public void should_stop_after_the_configured_number_of_addresses() { + // Given – a name expanding to three addresses, but a cap of two. Every address tried is a full + // connect plus handshake -- and, with wrong credentials, a rejected login -- and the + // reconnection fallback re-appends the contact points to every round, so an unbounded walk + // would repeat per contact point, per round, for as long as the session lives. The cap is what + // bounds that. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – only two addresses were dialed. The count of suppressed entries is not what to assert + // on: a single dead candidate can contribute two entangled failures (the transport refusal, + // plus an init-write failure that PromiseCombiner attaches to it as suppressed). The set of + // addresses named anywhere in the aggregate is what reflects the dials. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(mentionedUnreachableAddresses(e)) + .as("a cap of 2 means exactly two addresses dialed") + .hasSize(2)); + } + + private static final Pattern UNREACHABLE_NAME = Pattern.compile("unreachable-\\d"); + + /** The distinct dead-address names mentioned anywhere in {@code error}'s suppressed tree. */ + private static Set mentionedUnreachableAddresses(Throwable error) { + Set names = new HashSet<>(); + Deque toVisit = new ArrayDeque<>(); + toVisit.push(error); + while (!toVisit.isEmpty()) { + Throwable current = toVisit.pop(); + String message = current.getMessage(); + if (message != null) { + Matcher matcher = UNREACHABLE_NAME.matcher(message); + while (matcher.find()) { + names.add(matcher.group()); + } + } + for (Throwable suppressed : current.getSuppressed()) { + toVisit.push(suppressed); + } + } + return names; + } + + @Test + public void should_shuffle_candidates_without_losing_any() { + // The order is random per connect -- that is what spreads load across a name's records and + // varies the starting address between successive attempts -- but every address must survive + // the shuffle, since the loop's fallback walks this list. + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), true)) + .containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + } + + @Test + public void should_order_candidates_by_the_injected_random_source() { + // The injection point for ordering-sensitive tests: a seeded Random produces the same + // permutation on two factories, so a scenario that needs a particular order picks a seed + // instead of depending on a sort the production code no longer performs. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory one = newChannelFactory(); + ChannelFactory other = newChannelFactory(); + one.random = new Random(42); + other.random = new Random(42); + + assertThat(one.shuffleAndLimit(addresses, true)) + .containsExactlyElementsOf(other.shuffleAndLimit(addresses, true)); + } + + @Test + public void should_leave_a_single_address_alone() { + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Collections.singletonList(UNREACHABLE_1), true)) + .containsExactly(UNREACHABLE_1); + } + + @Test + public void should_truncate_the_shuffled_list_to_the_cap() { + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + + List capped = factory.shuffleAndLimit(addresses, true); + + assertThat(capped).hasSize(2); + assertThat(addresses).containsAll(capped); + } + + @Test + public void should_clamp_the_cap_to_at_least_one_address() { + // Zero or a negative value cannot mean "dial nothing" -- the attempt would fail without ever + // trying an address. It degrades to the pre-multi-address behavior of one address per attempt. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(0); + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2), true)) + .hasSize(1); + } + + @Test + public void should_not_shuffle_when_the_addresses_are_not_interchangeable() { + // A name that may denote different hosts -- what an AddressTranslator can hand back, and + // SubnetAddressTranslator does by default -- must keep the resolver's order: a random one would + // scatter a single Node's pool across hosts that routing, shard awareness and per-node metrics + // all attribute to that one node. Keeping the order makes such a pool converge on one address, + // as it did before multi-address support, while the rest of the list still serves as fallback. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory factory = newChannelFactory(); + // A seed that does permute this list, so the assertion below fails if the shuffle still runs. + factory.random = new Random(42); + assertThat(factory.shuffleAndLimit(addresses, true)).isNotEqualTo(addresses); + + assertThat(factory.shuffleAndLimit(addresses, false)).containsExactlyElementsOf(addresses); + } + + @Test + public void should_still_cap_the_candidates_when_the_order_is_kept() { + // The cap is what bounds the cost of one attempt, and that applies whether or not the addresses + // were shuffled. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), false)) + .containsExactly(UNREACHABLE_1, UNREACHABLE_2); + } + + // ---- addressesAreInterchangeable() and the two booleans derived from it ---- + + /** The endpoints below only have to exist; nothing in this section connects to them. */ + private static final InetSocketAddress SOME_ADDRESS = + InetSocketAddress.createUnresolved("node.example.com", 9042); + + @Test + public void should_report_a_proxy_endpoint_interchangeable() { + // An SNI proxy routes by server name, so every one of its A-records reaches the same node. + assertThat( + ChannelFactory.addressesAreInterchangeable( + new SniEndPoint(SOME_ADDRESS, "server-name"), SOME_ADDRESS)) + .isTrue(); + } + + @Test + public void should_not_report_a_plain_endpoint_interchangeable() { + // The case the flag exists to exclude: a DefaultEndPoint holding a name an AddressTranslator + // supplied carries no guarantee that its addresses are one server. + assertThat( + ChannelFactory.addressesAreInterchangeable( + new DefaultEndPoint(SOME_ADDRESS), SOME_ADDRESS)) + .isFalse(); + } + + @Test + public void should_not_report_a_third_party_endpoint_interchangeable() { + // An EndPoint that does not implement PinnableEndPoint cannot say, and the conservative reading + // is the one that assumes nothing. + EndPoint thirdParty = mock(EndPoint.class); + when(thirdParty.resolve()).thenReturn(SOME_ADDRESS); + + assertThat(ChannelFactory.addressesAreInterchangeable(thirdParty, SOME_ADDRESS)).isFalse(); + } + + @Test + public void should_spread_unless_an_identified_node_says_its_addresses_are_not_one_server() { + // A contact point always spreads: nothing is known about its addresses -- they may be + // different nodes -- so there is no node identity to preserve. + assertThat(ChannelFactory.spreadAcrossAddresses(false, false)).isTrue(); + assertThat(ChannelFactory.spreadAcrossAddresses(false, true)).isTrue(); + // An identified node spreads only where its addresses are interchangeable. + assertThat(ChannelFactory.spreadAcrossAddresses(true, true)).isTrue(); + assertThat(ChannelFactory.spreadAcrossAddresses(true, false)).isFalse(); + } + + @Test + public void should_treat_one_server_as_answering_everywhere_only_on_identity_or_interchange() { + // Not the negation of the above, and the difference is the whole of DRIVER-201's rolling- + // upgrade case: an unidentified contact point on a plain multi-record name is spread across + // its addresses *and* must not let one address's rejection speak for the others. + assertThat(ChannelFactory.sameServerAtEveryAddress(false, false)).isFalse(); + assertThat(ChannelFactory.sameServerAtEveryAddress(false, true)).isTrue(); + assertThat(ChannelFactory.sameServerAtEveryAddress(true, false)).isTrue(); + assertThat(ChannelFactory.sameServerAtEveryAddress(true, true)).isTrue(); + } + + // ---- reattachHostname() --------------------------------------------------- + + @Test + public void should_reattach_queried_hostname_to_nameless_resolved_address() throws Exception { + // A custom resolver may build its results from raw address bytes; the queried name must be + // re-attached so TLS hostname validation checks the configured name (not the IP or a PTR + // record) and reading the host name never triggers a reverse lookup on the event loop. + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9999); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.isUnresolved()).isFalse(); + // getHostString() never looks anything up; getHostName() reverse-resolves a *nameless* + // address, so it returning the queried name proves the name is embedded, not looked up. + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getHostName()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + // The candidate's port wins over the original's: a resolver may remap ports too. + assertThat(result.getPort()).isEqualTo(9999); + // Equality is unchanged (a resolved InetSocketAddress compares IP bytes + port only), so + // pinning and the pin-equality shortcuts behave exactly as with the raw candidate. + assertThat(result).isEqualTo(candidate); + } + + @Test + public void should_override_resolver_provided_hostname_with_queried_name() throws Exception { + // A resolver may label its results with a canonical/CNAME name of its own. That name would end + // up on the pinned endpoint and hence be the one TLS hostname verification checks the server + // certificate against, so the name the user configured has to win over it. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("cname.example.fake", new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_pass_candidate_through_when_it_already_carries_the_queried_name() + throws Exception { + // The common case: the JDK and Netty-DNS resolvers attach the queried name themselves, so + // there is nothing to rebuild. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("test.cluster.fake", new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + } + + @Test + public void should_pass_non_inet_candidate_through() { + // The local-transport addresses these unit tests connect over must never be touched. + assertThat(ChannelFactory.reattachHostname(HOSTNAME, UNREACHABLE_1)).isSameAs(UNREACHABLE_1); + } + + @Test + public void should_pass_redirected_candidate_through_when_original_is_an_ip_literal() + throws Exception { + // An original written as an IP literal has no name to carry over, and inventing one from the + // literal would be worse than leaving the candidate alone: a resolver is free to redirect it to + // a different IP, which would then be labelled with the literal form of a *different* address. + InetSocketAddress original = InetSocketAddress.createUnresolved("127.0.0.1", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + assertThat(AddressUtils.carriesName(original)).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) + .isFalse(); + } + + @Test + public void should_reattach_the_literal_when_the_resolver_returns_the_same_address() + throws Exception { + // Not a no-op, even though the label says the same thing the bytes do: a *nameless* address is + // what InetSocketAddress#getHostName() answers with a blocking reverse lookup, so leaving the + // candidate unlabelled is what would send DefaultSslEngineFactory to a PTR record instead of + // the + // literal the operator configured. Before multi-address support the contact point stayed + // unresolved and the literal came back with no lookup at all; labelling restores exactly that. + InetSocketAddress original = InetSocketAddress.createUnresolved("127.0.0.1", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(result.getHostString()).isEqualTo("127.0.0.1"); + // The point of the exercise: getHostName() is a field read answering the configured literal, + // not a reverse lookup. + assertThat(result.getAddress().getHostName()).isEqualTo("127.0.0.1"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("127.0.0.1"); + // And the labelled candidate still reports as a literal, so nothing downstream mistakes it for + // a name. + assertThat(AddressUtils.carriesName(result)).isFalse(); + } + + @Test + public void should_match_a_non_canonical_ipv6_literal_against_the_candidate() throws Exception { + // The literal is compared as an address, not as a string: "::1" and the candidate's + // getHostAddress() ("0:0:0:0:0:0:0:1") never compare equal as text. + InetSocketAddress original = InetSocketAddress.createUnresolved("::1", 9042); + byte[] loopback = new byte[16]; + loopback[15] = 1; + InetSocketAddress candidate = new InetSocketAddress(InetAddress.getByAddress(loopback), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(result.getHostString()).isEqualTo("::1"); + assertThat(result.getAddress()).isEqualTo(candidate.getAddress()); + } + + // ---- materializeLiteral() -------------------------------------------------- + + @Test + public void should_materialize_an_unresolved_ipv4_literal() { + // A literal needs no name service, so an endpoint holding one has no business failing where + // resolution is unavailable -- and endpoints hold one routinely now that contact points are + // kept unresolved whatever they were written as. + InetSocketAddress literal = InetSocketAddress.createUnresolved("127.0.0.1", 9042); + + InetSocketAddress result = (InetSocketAddress) ChannelFactory.materializeLiteral(literal); + + assertThat(result.isUnresolved()).isFalse(); + assertThat(result.getAddress().getAddress()).isEqualTo(new byte[] {127, 0, 0, 1}); + assertThat(result.getPort()).isEqualTo(9042); + // Labelled with the literal, not left nameless: getHostName() on a nameless address is a + // blocking reverse lookup, and DefaultSslEngineFactory would validate the certificate against + // whatever PTR record it returned instead of what the operator configured. + assertThat(result.getHostName()).isEqualTo("127.0.0.1"); + assertThat(AddressUtils.carriesName(result)).isFalse(); + } + + @Test + public void should_materialize_a_bracketed_ipv6_literal() { + // The spelling AddressUtils#extract preserves: it splits a contact point on its last colon, so + // "[::1]:9042" arrives with the brackets still on. + InetSocketAddress literal = InetSocketAddress.createUnresolved("[::1]", 9042); + + InetSocketAddress result = (InetSocketAddress) ChannelFactory.materializeLiteral(literal); + + byte[] loopback = new byte[16]; + loopback[15] = 1; + assertThat(result.isUnresolved()).isFalse(); + assertThat(result.getAddress().getAddress()).isEqualTo(loopback); + // Without the brackets: InetAddress.getByAddress(String, byte[]) strips them from the label it + // is handed. Still a literal, so getHostName() still answers without a reverse lookup, which is + // the only property this label exists for. + assertThat(result.getHostName()).isEqualTo("::1"); + } + + @Test + public void should_materialize_a_zoned_ipv6_literal() throws Exception { + // A named IPv6 zone is a literal to AddressUtils#carriesName, so it reaches here -- and unlike + // the byte-matching in reattachHostname, which resolves the zone through Guava, the JDK turns + // the name into a scope id itself. That costs a NetworkInterface syscall rather than a name + // lookup, which is the one place materializeLiteral is not free. + InetAddress linkLocal = aLinkLocalAddress(); + assumeThat(linkLocal) + .as("requires a host with a link-local IPv6 address on some interface") + .isNotNull(); + // Already the zoned spelling: getHostAddress() on a scoped address renders the zone as the + // interface name, e.g. "fe80:0:0:0:...%eth0". + String spelling = linkLocal.getHostAddress(); + assumeThat(spelling).as("expected a named zone").contains("%"); + + InetSocketAddress result = + (InetSocketAddress) + ChannelFactory.materializeLiteral(InetSocketAddress.createUnresolved(spelling, 9042)); + + assertThat(result).isNotNull(); + assertThat(result.isUnresolved()).isFalse(); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal.getAddress()); + assertThat(result.getPort()).isEqualTo(9042); + } + + /** + * A link-local IPv6 address of some interface on this host, or {@code null} if it has none. Only + * a zone that names an interface which actually carries an address in that scope survives {@code + * InetAddress.getByName}, so the address has to be discovered rather than made up. + */ + @Nullable + private static InetAddress aLinkLocalAddress() { + try { + for (NetworkInterface nic : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress address : Collections.list(nic.getInetAddresses())) { + if (address instanceof Inet6Address && address.isLinkLocalAddress()) { + return address; + } + } + } + } catch (SocketException unavailable) { + // Treated as "this host has none". + } + return null; + } + + @Test + public void should_not_materialize_a_zone_this_host_does_not_have() { + // The gate and the materializer disagree here: carriesName() calls it a literal, and + // InetAddress.getByName() cannot turn a name it has no interface for into a scope id. Falling + // through to the caller's diagnostic is the right outcome -- such an address could not have + // been connected to either -- but its wording is about host names, which this is not. Recorded + // so that the mismatch is a known one rather than a surprise. + assertThat( + ChannelFactory.materializeLiteral( + InetSocketAddress.createUnresolved("fe80::1%no-such-interface", 9042))) + .isNull(); + } + + @Test + public void should_not_materialize_a_shorthand_ipv4_literal() { + // "127.1" is /127.0.0.1 to InetAddress.getByName and to Netty's default resolver, but Guava's + // parser requires four dotted parts, so carriesName() calls it a host name and this returns at + // the first gate. Deliberate: getByName("1234") returns /0.0.4.210, so a test as lenient as + // the JDK's would turn an all-digit host name into a packed IPv4 address. The cost is that + // such a contact point works normally and fails only where no resolver runs. + assertThat(ChannelFactory.materializeLiteral(InetSocketAddress.createUnresolved("127.1", 9042))) + .isNull(); + } + + @Test + public void should_not_materialize_a_hostname() { + // The whole point of the diagnostic this sits in front of: a name genuinely needs a resolver, + // and passing it through would fail later inside Netty with UnresolvedAddressException, naming + // neither the address nor the reason. + assertThat( + ChannelFactory.materializeLiteral( + InetSocketAddress.createUnresolved("node.example.com", 9042))) + .isNull(); + } + + @Test + public void should_not_materialize_an_already_resolved_address() throws Exception { + // Nothing to do; the caller passes it through untouched. + assertThat( + ChannelFactory.materializeLiteral( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042))) + .isNull(); + } + + @Test + public void should_reattach_the_name_of_an_already_resolved_original() throws Exception { + // A resolved original reaches this only because a custom resolver reported it as unresolved in + // order to redirect it, and its name is re-attached like any other. `new + // InetSocketAddress(String, int)` resolves eagerly and keeps the name it was given, so this is + // the shape an AddressTranslator or a third-party EndPoint hands over -- and leaving the + // redirected candidate nameless is not neutral: DefaultSslEngineFactory would then take the TLS + // peer host from a blocking reverse lookup and validate the certificate against a PTR record + // instead of the configured DNS SAN, which is not what the pre-multi-address path did. + InetSocketAddress original = new InetSocketAddress("localhost", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + assertThat(original.isUnresolved()).isFalse(); + assertThat(AddressUtils.carriesName(original)).isTrue(); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(result.getHostString()).isEqualTo("localhost"); + assertThat(result.getAddress().getAddress()).isEqualTo(new byte[] {10, 0, 0, 1}); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_leave_a_resolved_original_alone_when_it_carries_no_name() throws Exception { + // The other half: a resolved original whose InetAddress has no cached hostName renders the IP + // literal, so it takes the literal branch and only matches the address it denotes. A redirect + // stays unlabelled rather than being given a name that resolves elsewhere. + InetSocketAddress original = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 2}), 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + assertThat(AddressUtils.carriesName(original)).isFalse(); + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + } + + @Test + public void should_reattach_hostname_to_nameless_ipv6_address() throws Exception { + byte[] loopback = new byte[16]; + loopback[15] = 1; // ::1 + InetSocketAddress candidate = new InetSocketAddress(InetAddress.getByAddress(loopback), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isEqualTo(candidate.getAddress()); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_label_a_candidate_from_a_bracketed_ipv6_literal_original() throws Exception { + // A contact point written "[2001:db8::5]:9042" reaches here with its brackets on: extract() + // splits on the last colon and keeps everything before it. carriesName() classifies that as a + // literal, so this takes the IP-literal branch -- and the branch has to unwrap the brackets + // before parsing, because InetAddresses.forString rejects the bracketed form outright. + // Failing to parse would return the candidate unlabelled and hand getHostName() a reverse + // lookup, which is the outcome the branch exists to prevent. + byte[] bytes = InetAddress.getByName("2001:db8::5").getAddress(); + InetSocketAddress original = InetSocketAddress.createUnresolved("[2001:db8::5]", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(null, bytes), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + // Labelled with the literal, and with no lookup. The brackets are gone because + // InetAddress.getByAddress(String, byte[]) strips a surrounding pair from the name it is + // given -- which is the canonical outcome: getHostString() now answers a bare literal, so + // carriesName() reports it as a literal on the way back too. + assertThat(result.getHostString()).isEqualTo("2001:db8::5"); + assertThat(result.getAddress().getAddress()).isEqualTo(bytes); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_not_label_a_redirected_candidate_from_a_bracketed_original() throws Exception { + // The redirect guard has to survive the unwrapping: a candidate that is not the address the + // bracketed literal denotes must come back unlabelled. Before brackets were recognised this + // case took the name-wins branch instead, which relabels unconditionally. + InetSocketAddress original = InetSocketAddress.createUnresolved("[2001:db8::5]", 9042); + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress(null, InetAddress.getByName("2001:db8::6").getAddress()), + 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + } + + @Test + public void should_label_a_candidate_from_a_bracketed_and_zoned_ipv6_literal_original() + throws Exception { + // Brackets *and* a zone: the brackets have to come off first, or splitting on '%' leaves the + // closing bracket inside the zone and the opening one inside the literal, and neither half + // parses. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress original = InetSocketAddress.createUnresolved("[fe80::1%eth0]", 9042); + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + // Bare literal with the zone intact -- getByAddress() strips only the brackets. + assertThat(result.getHostString()).isEqualTo("fe80::1%eth0"); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + } + + @Test + public void should_label_a_candidate_from_a_zoned_ipv6_literal_original() throws Exception { + // The original is a *literal* with a zone, which carriesName() reports as a literal (Guava's + // isInetAddress accepts a zone suffix), so reattachHostname takes its IP-literal branch. That + // branch cannot hand the string to InetAddresses.forString: Guava resolves the zone against the + // local interfaces and throws when it does not name one -- it rejects even "%lo" on a host that + // has an lo interface. Failing there would return the candidate unlabelled, and getHostName() + // would then answer with a reverse lookup, which is precisely what this branch exists to stop. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress original = InetSocketAddress.createUnresolved("fe80::1%eth0", 9042); + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + // Labelled with the literal exactly as configured, zone included, and with no lookup. + assertThat(result.getHostString()).isEqualTo("fe80::1%eth0"); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_not_label_a_candidate_that_is_a_different_address_from_a_zoned_original() + throws Exception { + // The redirect guard still has to hold on the zoned path: a candidate that is not the address + // the literal denotes must come back unlabelled. + byte[] other = new byte[16]; + other[0] = (byte) 0xfe; + other[1] = (byte) 0x80; + other[15] = 2; + InetSocketAddress original = InetSocketAddress.createUnresolved("fe80::1%eth0", 9042); + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, other, 3), 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + } + + @Test + public void should_keep_the_scope_when_reattaching_to_a_scoped_ipv6_address() throws Exception { + // A link-local address only points anywhere together with its zone, so the queried name has to + // be re-attached without dropping the scope. InetAddress.getByAddress(host, bytes) cannot carry + // one, but Inet6Address.getByAddress(host, bytes, scopeId) can. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isInstanceOf(Inet6Address.class); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(3); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_zone_of_an_interface_scoped_ipv6_address() throws Exception { + // An address built from a NetworkInterface rather than from an index must keep pointing into + // the + // same zone. The numeric scope the JDK derived at construction is what the connect goes on, so + // carrying that over is enough; only the interface name, a toString() detail, is not. + Inet6Address linkLocal = firstInterfaceScopedIpv6Address(); + assumeThat(linkLocal).as("no interface-scoped IPv6 address on this host").isNotNull(); + InetSocketAddress candidate = new InetSocketAddress(linkLocal, 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(linkLocal.getScopeId()); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal.getAddress()); + } + + /** An interface-scoped IPv6 address of this host, or null if it has none. */ + private static Inet6Address firstInterfaceScopedIpv6Address() throws Exception { + for (NetworkInterface nif : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress address : Collections.list(nif.getInetAddresses())) { + if (address instanceof Inet6Address + && ((Inet6Address) address).getScopedInterface() != null) { + return (Inet6Address) address; + } + } + } + return null; + } + + @Test + public void should_fail_future_when_endpoint_resolve_throws() { + // ChannelFactory calls EndPoint.resolve() directly on the caller thread, so a third-party + // implementation that throws must surface as a failed future rather than an escaping exception. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + IllegalStateException failure = new IllegalStateException("resolve() blew up"); + + CompletionStage channelFuture = + factory.connect( + new ThrowingEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + @Test + public void should_fail_future_when_addresses_are_interchangeable_throws() { + // The other implementation-supplied method connect() calls synchronously, and the one that is + // easy to miss: it decides whether the resolved addresses may be shuffled. Escaping here would + // be worse than escaping from resolve(), because ControlConnection#reconnect neither wraps its + // connect() call nor catches inside the whenCompleteAsync callback that drives the recursive + // ones -- the throwable would be swallowed and Reconnection left stuck ATTEMPT_IN_PROGRESS, + // with no further attempts. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + IllegalStateException failure = + new IllegalStateException("addressesAreInterchangeable() blew up"); + + CompletionStage channelFuture = + factory.connect( + new ThrowingSpreadEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + // Either value would do: the endpoint is consulted on every connect, because both of + // the booleans derived from its answer need it -- an unidentified contact point still + // has to know whether one address's rejection speaks for the rest. + false); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + @Test + public void should_fail_future_when_endpoint_spread_check_throws_an_error() { + // The guard catches Throwable, not Exception. An endpoint supplied by someone else can fail + // with an Error just as readily as with an exception -- NoClassDefFoundError or + // ExceptionInInitializerError out of lazy class initialization in a shaded or OSGi deployment, + // AssertionError under -ea -- and the outcome of letting one escape is the same hang: nothing + // upstream completes the future, so the attempt sits with Reconnection stuck in + // ATTEMPT_IN_PROGRESS and no further attempt is ever scheduled. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + Error failure = new NoClassDefFoundError("com/example/CustomEndPointSupport"); + + CompletionStage channelFuture = + factory.connect( + new ThrowingSpreadEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + @Test + public void should_fail_future_when_endpoint_resolve_returns_null() { + // EndPoint.resolve() is contractually non-null, but a broken third-party implementation must + // fail fast rather than NPE later inside an event-loop task, which would leave the future + // hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new NullResolvingEndPoint(), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("returned null")); + } + + @Test + public void should_fail_future_when_event_loop_group_is_rejecting_tasks() + throws InterruptedException { + // Resolution is dispatched to an I/O event loop; if the group is already shutting down, that + // dispatch is rejected synchronously. The rejection must fail the future rather than escape to + // the caller (connect() never used to throw) or leave the future hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + clientGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).sync(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).isInstanceOf(RejectedExecutionException.class)); + } + + /** An endpoint whose {@link EndPoint#resolve()} throws, standing in for a broken third party. */ + private static class ThrowingEndPoint implements EndPoint { + + private final RuntimeException failure; + + ThrowingEndPoint(RuntimeException failure) { + this.failure = failure; + } + + @NonNull + @Override + public SocketAddress resolve() { + throw failure; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } + + /** + * A {@link PinnableEndPoint} whose {@code addressesAreInterchangeable()} throws, standing in for + * any implementation-supplied override that can fail -- {@code ClientRoutesEndPoint}'s reaches + * the topology monitor and catches only {@link IllegalStateException}. + */ + private static class ThrowingSpreadEndPoint implements PinnableEndPoint { + + private final Throwable failure; + + ThrowingSpreadEndPoint(Throwable failure) { + this.failure = failure; + } + + @NonNull + @Override + public SocketAddress resolve() { + return InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + } + + @Override + public boolean addressesAreInterchangeable(@NonNull SocketAddress resolvedAddress) { + if (failure instanceof Error) { + throw (Error) failure; + } + throw (RuntimeException) failure; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + return this; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } + + /** A broken third-party endpoint that violates {@code resolve()}'s non-null contract. */ + private static class NullResolvingEndPoint implements EndPoint { + + @NonNull + @Override + @SuppressWarnings("NullAway") // deliberately broken, that is the point of the test + public SocketAddress resolve() { + return null; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java new file mode 100644 index 00000000000..8c9c7e98b5d --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.DefaultEventLoopGroup; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory} expands unresolved candidate addresses through Netty's + * configured {@link AddressResolverGroup}, rather than doing its own JVM DNS lookup. + * + *

This is what keeps a custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)} + * effective: before multi-address support, an unresolved address was handed straight to {@code + * Bootstrap.connect()} and Netty's resolver expanded it, so resolving anywhere else would silently + * bypass the user's configuration. + */ +public class ChannelFactoryNettyResolverTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryNettyResolverTest.class.getSimpleName() + "-unreachable"); + + /** The hostname the endpoint reports, and that only the custom resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + /** What a resolver must never hand back from {@code resolveAll}, but might. */ + private static final InetSocketAddress STILL_UNRESOLVED = + InetSocketAddress.createUnresolved("still.unresolved.fake", 9042); + + @Test + public void should_expand_unresolved_address_through_the_custom_netty_resolver() { + // Given – a resolver that maps the hostname to an unreachable address followed by the running + // local server, mimicking a DNS round-robin entry whose first record is dead. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + // Keeping the resolver's order is what makes success here mean anything: it puts the dead + // record first, so the connect can only succeed by falling back to the second address. Left to + // the production shuffle the dialled order is a coin flip for a two-element list, and this test + // would pass about half the time even with the fallback in tryNextCandidate() broken -- those + // runs simply dial the reachable address first and never exercise it. + factory.random = new KeepResolverOrder(); + + // When – the endpoint itself performs no resolution at all; it just yields the hostname. + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second address. + completeSimpleChannelInit(); + + // Then – the custom resolver was consulted for the hostname, and the dead first record did + // not end the attempt: the connection survived it by trying the address behind it. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the custom Netty resolver must be the one expanding the hostname") + .containsExactly(HOSTNAME); + } + + @Test + public void should_fail_when_the_custom_resolver_cannot_resolve_the_only_candidate() { + // Given – a resolver that fails every lookup. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = new TestAddressResolverGroup(null); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – no candidate survived resolution, so the connect fails with the resolver's own cause + // rather than, say, an empty-candidate-list error. + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).hasMessageContaining("mock resolver failure")); + } + + @Test + public void should_fail_with_a_diagnosable_error_when_every_expanded_address_is_unresolved() { + // Given – a resolver that "expands" the hostname to another unresolved address. A redirecting + // resolver can do this by rewriting the host without resolving it, and nothing downstream will + // resolve it either: connectToAddress() uses a bootstrap clone with disableResolver(), so Netty + // would raise UnresolvedAddressException from inside doConnect, naming neither the address nor + // the reason nothing resolved it -- for every connect of the whole session. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Collections.singletonList(STILL_UNRESOLVED))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the failure says which endpoint and which resolver produced it, as the pass-through + // paths already did (see ChannelFactory#unusableWithoutResolution). + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(IllegalStateException.class); + assertThat(e.getMessage()).contains("test.cluster.fake"); + assertThat(e.getMessage()).contains("TestAddressResolverGroup"); + assertThat(e.getMessage()).contains("unresolved"); + }); + } + + @Test + public void should_drop_an_unresolved_expanded_address_before_applying_the_cap() { + // Given – the same resolver answering with one unusable address and one live one, and a cap of + // a + // single candidate. An address that cannot be connected to must not consume a slot in that cap: + // dropped after the truncation instead, it would leave this connect with nothing to dial. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(1); + installResolver( + new TestAddressResolverGroup(Arrays.asList(STILL_UNRESOLVED, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // Keeping the resolver's order is what makes this an assertion about the cap rather than about + // luck: the unusable address is the one the truncation would otherwise have kept. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { + // Given – Bootstrap.disableResolver() means config().resolver() is null. ChannelFactory must + // treat that as "pass the candidates through" instead of dereferencing the missing group. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(resolverGroup).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint yields an already-usable address. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – connection succeeds and the resolver was never even instantiated, let alone consulted. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.resolverRequested).isFalse(); + assertThat(resolverGroup.queried).isEmpty(); + } + + @Test + public void should_materialize_an_ip_literal_when_the_user_disabled_the_resolver() { + // Given – disableResolver() and an endpoint holding an unresolved IP literal, which is now the + // ordinary shape: contact points are kept unresolved whatever they were written as. Before + // that they arrived here already resolved and disableResolver() worked with them, and a + // literal needs no name service for that to stay true. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(resolverGroup).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("127.0.0.1", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – it got as far as dialling the literal. The transport in this harness is Netty's local + // one, so an InetSocketAddress has nothing bound to it and the connect is refused by name -- + // which is the point: the attempt failed at the socket, not at the "nothing will resolve this" + // diagnostic it used to die on before reaching one. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isNotInstanceOf(IllegalStateException.class); + assertThat(e).hasMessageContaining("127.0.0.1"); + }); + assertThat(resolverGroup.resolverRequested).isFalse(); + } + + @Test + public void should_still_fail_a_hostname_when_the_user_disabled_the_resolver() { + // The other half of the same branch: a name genuinely needs a resolver, so it keeps failing -- + // with a message that names disableResolver() as the cause, since that is what the operator + // has to undo. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap + .resolver(new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE))) + .disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(IllegalStateException.class); + assertThat(e.getMessage()) + .contains("test.cluster.fake") + .contains("the bootstrap has name resolution disabled"); + }); + } + + @Test + public void should_pass_a_declined_address_through_untouched() { + // Given – a resolver that declines every address, as a real one does for an address type it + // does + // not handle (DefaultNameResolver declines anything that is not an InetSocketAddress). Netty + // passes such an address through in Bootstrap#doResolveAndConnect0, and so must the driver. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE), false, true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint yields an already-usable address. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – it connected to the address as given; nothing was looked up or substituted. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried).isEmpty(); + } + + @Test + public void should_report_a_declined_unresolved_address_as_such() { + // Given – the same declining resolver, but now the address needs resolving. Nothing downstream + // will do it (connectToAddress uses a bootstrap clone with disableResolver()), so this fails + // every connection attempt for the session and the message has to name the actual cause. Naming + // the disabled-resolver case instead would send the operator looking for a + // Bootstrap.disableResolver() nobody called. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver( + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE), false, true)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(IllegalStateException.class); + assertThat(e.getMessage()) + .contains("test.cluster.fake") + .contains("the configured resolver does not support this address") + .doesNotContain("disableResolver"); + }); + } + + @Test + public void should_pass_already_resolved_address_through_untouched() { + // Given – an endpoint whose address is already resolved, which is the common case: metadata + // nodes hold resolved addresses from the peers rows, so this is every pool refill and every + // reconnect. A resolver with the usual semantics reports it as resolved and there is nothing + // to expand. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – no lookup was performed: had one been, it would have redirected us to UNREACHABLE and + // the connection would have failed. The decision was the resolver's own, though -- see + // should_let_the_resolver_redirect_an_already_resolved_address. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried).isEmpty(); + assertThat(resolverGroup.resolverRequested) + .as("whether an address needs resolving must be the resolver's decision") + .isTrue(); + } + + @Test + public void should_pass_a_name_through_when_the_resolver_claims_it_is_resolved() { + // Given – NoopAddressResolverGroup's shape: supports every address, reports every address + // resolved. That is not a broken resolver, it is Netty's documented way of handing name + // resolution to something in the pipeline -- a ProxyHandler added through + // NettyOptions.afterChannelInitialized(), which intercepts the connect and sends the name on + // to the proxy. Netty's own Bootstrap#doResolveAndConnect0 short-circuits on exactly this and + // calls doConnect() with the address untouched. + // + // Refusing it would fail *every* connect of the session, contact points now being kept + // unresolved whatever they were written as -- so 127.0.0.1:9042 would die here too. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + TestAddressResolverGroup.claimingEverythingIsResolved(); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the connect was attempted with the name as-is. It cannot succeed here: these tests run + // over Netty's local transport, which has no server bound to an InetSocketAddress, so the + // failure comes from the transport. What matters is which failure -- withholding the + // pass-through fails it in resolveCandidates() instead, with the IllegalStateException that + // tells the operator to fix their resolver. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .as("the resolver's claim must be honoured, as Netty honours it") + .isNotInstanceOf(IllegalStateException.class)); + assertThat(resolverGroup.queried) + .as("an address the resolver called resolved must not then be resolved") + .isEmpty(); + } + + @Test + public void should_let_the_resolver_redirect_an_already_resolved_address() { + // Given – a resolver that reports even an address carrying an IP as still needing resolution, + // and redirects it. Netty consulted the resolver for every connect, resolved address or not + // (Bootstrap#doResolveAndConnect0 calls isSupported()/isResolved() on it rather than testing + // the address itself), so short-circuiting on InetSocketAddress#isUnresolved() here would take + // that away for every connect to an already-resolved node -- which is nearly all of them. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Collections.singletonList(SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint holds a resolved address that nothing is listening on. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(resolved), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the connect landed on the address the resolver substituted, which it could only do by + // having been asked about an address that already carried an IP. Exactly one lookup: the + // per-attempt bootstrap has the resolver disabled, so the substitute is connected to as-is + // rather than being handed back to the resolver (see the next test for why that matters). + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the resolver must get a say on an address that already carries an IP") + .containsExactly(resolved); + } + + @Test + public void should_try_every_candidate_when_the_resolver_redirects() { + // Given – the same redirecting resolver as above, but answering with more than one address: + // a dead one first, then the running local server. + // + // The per-attempt bootstrap must not re-resolve. Bootstrap.clone() carries the resolver + // configuration over, and Netty's own pass calls resolve() -- *singular* -- so with a resolver + // that reports resolved addresses as unresolved, every candidate would be redirected again onto + // the resolver's first answer: the dead address, N times over. Multi-address fallback would + // silently do nothing, and the endpoint pinned onto the channel would name an address the + // channel is not connected to -- which is what the SSL engine's peer host and + // DefaultTopologyMonitor#savePort are then derived from. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the reachable address was actually reached. This holds whichever candidate rotate() + // starts from, and it is precisely what fails when the clone re-resolves: the dead address is + // the resolver's first answer, so both attempts would land there and the connect would fail. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the hostname is expanded once, by ChannelFactory; the candidates are not re-resolved") + .containsExactly(HOSTNAME); + } + + @Test + public void should_resolve_and_connect_on_the_same_event_loop() throws InterruptedException { + // Resolution and channel registration must share the loop picked once per connect. Taking one + // loop for resolution and letting the registration pick another would advance the group's + // round-robin chooser twice per connect, parking every channel on half the loops with the + // default power-of-two chooser. The base's single-thread group would make this assertion + // vacuous, so use two loops -- on which the split behavior was deterministic. + DefaultEventLoopGroup twoLoops = new DefaultEventLoopGroup(2); + try { + when(nettyOptions.ioEventLoopGroup()).thenReturn(twoLoops); + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess( + channel -> + assertThat((Object) channel.eventLoop()) + .as("the channel must be registered on the loop resolution ran on") + .isSameAs(resolverGroup.resolverExecutor)); + } finally { + twoLoops.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS).sync(); + } + } + + @Test + public void should_fail_future_when_resolver_throws_synchronously() { + // Given – a broken custom resolver that throws instead of returning a failed future. The throw + // happens inside an event-loop task, where nothing else would ever complete the connect future: + // nothing at this stage has a timeout, so before the blanket catch in resolveCandidates() this + // hung the connect attempt (and with it control-connection init) forever. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("broken resolver"); + installResolver(new ThrowingAddressResolverGroup(failure)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** A resolver whose every method throws, standing in for a broken third-party implementation. */ + private static class ThrowingAddressResolverGroup extends AddressResolverGroup { + + private final RuntimeException failure; + + ThrowingAddressResolverGroup(RuntimeException failure) { + this.failure = failure; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + throw failure; + } + + @Override + public boolean isResolved(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve( + SocketAddress address, Promise promise) { + throw failure; + } + + @Override + public Future> resolveAll(SocketAddress address) { + throw failure; + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + throw failure; + } + + @Override + public void close() { + // nothing to do + } + }; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java new file mode 100644 index 00000000000..b313edb0d1f --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies that a successfully connected {@link DriverChannel} carries an endpoint bound to the + * address the connection actually used, not the multi-address original. + * + *

Without this, a hostname shared by several nodes would let a later reconnect land on a + * different node while still being treated as the original {@code host_id}: {@code + * DefaultTopologyMonitor#buildNodeEndPoint} stores the channel's endpoint for the control node, and + * {@code ControlConnection} skips identity re-resolution for nodes that already have a host id. See + * {@link PinnableEndPoint}. + */ +public class ChannelFactoryPinnedEndPointTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryPinnedEndPointTest.class.getSimpleName() + "-unreachable"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_pin_channel_endpoint_to_the_address_that_connected() { + // Given – an endpoint reporting a name, which the resolver expands to a dead address and the + // running local server. Whichever of the two the connection ends up on, the channel must carry + // that one. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); + ChannelFactory factory = newChannelFactory(); + TestPinnableEndPoint endPoint = new TestPinnableEndPoint(HOSTNAME); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture) + .isSuccess( + channel -> { + EndPoint channelEndPoint = channel.getEndPoint(); + // The channel resolves to the address it is actually connected to -- the name it was + // built from is gone from resolve(), which is what SSL engines and authenticators + // need. + assertThat(channelEndPoint.resolve()).isEqualTo(reachable); + // ...while still denoting the same node, so node lookups and metric names are stable. + assertThat(channelEndPoint).isEqualTo(endPoint); + assertThat(channelEndPoint.asMetricPrefix()).isEqualTo(endPoint.asMetricPrefix()); + }); + } + + @Test + public void should_leave_non_pinnable_endpoints_untouched() { + // A third-party EndPoint that does not implement PinnableEndPoint must reach the channel + // exactly + // as it was given, so existing implementations keep working. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel.getEndPoint()).isSameAs(SERVER_ADDRESS)); + } + + @Test + public void should_not_pin_to_an_unresolved_address() { + // Given – a resolver that reports every address already resolved, which is what + // NoopAddressResolverGroup does when something in the pipeline resolves the name instead. + // resolveCandidates() honours the claim and hands the endpoint's own address straight back, so + // for an endpoint that reports a name the candidate is still that name. This is the only path + // that reaches pin() with an unresolved address: the other two pass-throughs materialize an IP + // literal or fail, and resolveAll's results have the unresolved ones dropped. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(TestAddressResolverGroup.claimingEverythingIsResolved()); + ChannelFactory factory = newChannelFactory(); + List pinnedTo = new ArrayList<>(); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + pinnedTo.add(resolvedAddress); + return super.pinTo(resolvedAddress); + } + }; + + // When – the connect itself cannot succeed: these tests run over Netty's local transport, which + // has no server bound to an InetSocketAddress. pin() runs before the attempt, so what it was + // handed is settled either way. + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – pinTo() is documented to take an already-resolved address. Pinning a name would freeze + // the endpoint on something that still re-expands on every connect, and for endpoints that stop + // consulting their own source once pinned, silence that source for good. + assertThatStage(channelFuture).isFailed(); + assertThat(pinnedTo).as("pinTo() must not be handed an unresolved address").isEmpty(); + } + + @Test + public void should_fail_future_when_pin_to_throws() { + // pinTo() runs in the continuation after resolution, whose exceptions CompletionStage + // swallows; a throwing implementation must fail the connect future rather than hang it. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Collections.singletonList(reachable))); + ChannelFactory factory = newChannelFactory(); + RuntimeException failure = new IllegalStateException("pinTo blew up"); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + throw failure; + } + }; + + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** + * A {@link PinnableEndPoint} that can hold a pin to any {@link SocketAddress}, including the + * local-transport addresses these tests connect over (which {@code DefaultEndPoint} cannot). + * Identity is the unpinned address, so a pinned copy stays equal to the original — the contract + * {@link PinnableEndPoint} requires. + */ + private static class TestPinnableEndPoint implements PinnableEndPoint { + + private final SocketAddress address; + private final SocketAddress pinnedAddress; + + TestPinnableEndPoint(SocketAddress address) { + this(address, null); + } + + private TestPinnableEndPoint(SocketAddress address, SocketAddress pinnedAddress) { + this.address = address; + this.pinnedAddress = pinnedAddress; + } + + @NonNull + @Override + public SocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : address; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + return new TestPinnableEndPoint(address, resolvedAddress); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + + @Override + public boolean equals(Object other) { + return (other instanceof TestPinnableEndPoint) + && address.equals(((TestPinnableEndPoint) other).address); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index fceb8777904..f6e39d1962b 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -33,12 +34,21 @@ import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletionStage; import org.junit.Test; public class ChannelFactoryProtocolNegotiationTest extends ChannelFactoryTestBase { + /** A local address no server is bound to: connecting to it fails immediately. */ + private static final SocketAddress UNREACHABLE = + new LocalAddress( + ChannelFactoryProtocolNegotiationTest.class.getSimpleName() + "-unreachable"); + @Test public void should_succeed_if_version_specified_and_supported_by_server() { // Given @@ -61,6 +71,11 @@ public void should_succeed_if_version_specified_and_supported_by_server() { // Then assertThatStage(channelFuture) .isSuccess(channel -> assertThat(channel.getClusterName()).isEqualTo("mockClusterName")); + // Read directly, not awaited: completeCandidate() records the negotiated state *before* it + // publishes the channel, so the value is already there for anyone the completed future reaches + // -- inline dependents included. Reading it straight is what pins that ordering. Awaiting would + // pass either way, and used to: while the latch still ran after the publish, an inline read + // here could see the pre-latch null, which is how this flaked on JDK 17 in CI. assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V4); } @@ -176,6 +191,11 @@ public void should_succeed_if_version_not_specified_and_server_supports_latest_s // Then assertThatStage(channelFuture) .isSuccess(channel -> assertThat(channel.getClusterName()).isEqualTo("mockClusterName")); + // Read directly, not awaited: completeCandidate() records the negotiated state *before* it + // publishes the channel, so the value is already there for anyone the completed future reaches + // -- inline dependents included. Reading it straight is what pins that ordering. Awaiting would + // pass either way, and used to: while the latch still ran after the publish, an inline read + // here could see the pre-latch null, which is how this flaked on JDK 17 in CI. assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V4); } @@ -222,6 +242,11 @@ public void should_negotiate_if_version_not_specified_and_server_supports_legacy writeInboundFrame(requestFrame, TestResponses.clusterNameResponse("mockClusterName")); assertThatStage(channelFuture) .isSuccess(channel -> assertThat(channel.getClusterName()).isEqualTo("mockClusterName")); + // Read directly, not awaited: completeCandidate() records the negotiated state *before* it + // publishes the channel, so the value is already there for anyone the completed future reaches + // -- inline dependents included. Reading it straight is what pins that ordering. Awaiting would + // pass either way, and used to: while the latch still ran after the publish, an inline read + // here could see the pre-latch null, which is how this flaked on JDK 17 in CI. assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V3); } @@ -280,6 +305,213 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) }); } + @Test + public void should_not_try_next_address_of_identified_node_when_negotiation_exhausts_versions() { + // Given – an *identified* node (its host id is known, so every address its name expands to is + // that same node) whose name expands to two candidates: the same live server twice, so + // whichever the rotation picks first is irrelevant. The server rejects every protocol version. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + exhaustNegotiationLadder(); + + // Then – the second candidate must not be attempted: for a node we have already identified, a + // protocol-version rejection is a property of the node, not of the address, so replaying the + // negotiation ladder against the remaining IPs would buy nothing. Checked before the future + // assertion so that on regression the stray frame is drained; leaving it unread would block the + // server's exchanger and hang the whole suite in tearDown() instead of failing this test. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after negotiation exhaustion") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + + @Test + public void + should_try_next_address_of_unidentified_endpoint_when_negotiation_exhausts_versions() { + // Given – the same setup, but for an endpoint the driver has not identified yet: a contact + // point, before host ids have been read. Its name may well expand to addresses of *different* + // nodes, so a version rejection by the first says nothing about the second. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + false); + + // The first candidate exhausts the ladder... + exhaustNegotiationLadder(); + // ...and the second is tried all the same, replaying the ladder from the top. Before this, + // resolve-contact-points=true made each address a separate node and ControlConnection advanced + // to the next one on exactly this error; collapsing a name into one node must not lose that. + exhaustNegotiationLadder(); + + // Then + assertThat(tryReadOutboundFrame(200)) + .as("the name expands to two addresses, so there is no third attempt") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .as("each candidate negotiates on its own, so this is the last one's ladder") + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the first candidate's failure must still be reported") + .hasSize(1); + assertThat(e.getSuppressed()[0]) + .isInstanceOf(UnsupportedProtocolVersionException.class); + }); + } + + @Test + public void should_surface_the_node_wide_rejection_when_an_earlier_address_failed_on_transport() { + // Given – an identified node whose name expands to a dead address first and to the live server + // second, the server rejecting every protocol version. Every address of an identified node is + // that same node, so the rejection is a property of the node rather than of the address. + mockNegotiationLadderDownToV3(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // The order matters here, and only here: it is what makes the pass end with a *mixed* set of + // failures instead of the version rejection on its own. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + // The dead address sends nothing, so the ladder below is the second candidate's. + exhaustNegotiationLadder(); + + // Then – the version rejection is what the caller must see, even though the pass also produced + // a + // transport failure that ChannelPool#handleError does not classify and would otherwise be + // preferred (see ChannelFactory#surfacedFailure: a fatal failure needs every address to agree). + // A node-wide failure is the exception to that rule: demoting it would turn the forced-down + // node + // that a single-address connect has always produced into a plain reconnect. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the earlier address's transport failure should still be attached") + .hasSize(1); + }); + } + + /** Negotiation starts at V4 and has exactly one downgrade available, to V3. */ + private void mockNegotiationLadderDownToV3() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) + .thenReturn(Optional.of(DefaultProtocolVersion.V3)); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + } + + /** + * Plays the server side of a full negotiation ladder against one candidate address: V4 rejected, + * downgrade retry with V3 rejected, i.e. no version left to try on that address. + */ + private void exhaustNegotiationLadder() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V3.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + } + + @Test + public void should_fail_future_when_downgrade_lookup_throws_in_connect_listener() { + // Given – a version registry that throws when the factory looks up the downgrade. The lookup + // runs inside the Netty connect listener, which swallows throwables: without the blanket catch + // in connectToAddress() the connect future would never complete and the attempt would hang. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("registry broken"); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)).thenThrow(failure); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + // Server does not support v4, which is what sends the factory to the downgrade lookup + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + /** * Depending on the Cassandra version, an "unsupported protocol" response can use different error * codes, so we test all of them. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index ed6668a6c83..dfc6ba5e141 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -20,6 +20,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.ProtocolVersion; @@ -42,6 +44,7 @@ import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -53,9 +56,14 @@ import io.netty.channel.DefaultEventLoopGroup; import io.netty.channel.local.LocalChannel; import io.netty.channel.local.LocalServerChannel; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import java.net.SocketAddress; import java.time.Duration; import java.util.Collections; import java.util.Optional; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Exchanger; import java.util.concurrent.TimeUnit; @@ -89,6 +97,12 @@ public abstract class ChannelFactoryTestBase { DefaultEventLoopGroup serverGroup; DefaultEventLoopGroup clientGroup; + // Separate from clientGroup, as in production: this is where the shard-aware port scan is + // dispatched, and it must not be the channel's own I/O loop. + DefaultEventLoopGroup adminGroup; + // Where the connect-hook timeout is armed -- deliberately neither of the two groups above, since + // the hook runs on one of them and the blocking port scan on the other. + Timer timer; @Mock InternalDriverContext context; @Mock DriverConfig driverConfig; @@ -114,6 +128,10 @@ public void setup() throws InterruptedException { serverGroup = new DefaultEventLoopGroup(1); clientGroup = new DefaultEventLoopGroup(1); + adminGroup = new DefaultEventLoopGroup(1); + // A short tick so that stop() -- which joins the worker thread, and runs for every test in + // every subclass of this base -- does not wait out a default 100ms one. + timer = new HashedWheelTimer(10, TimeUnit.MILLISECONDS); when(context.getConfig()).thenReturn(driverConfig); when(driverConfig.getDefaultProfile()).thenReturn(defaultProfile); @@ -123,6 +141,9 @@ public void setup() throws InterruptedException { when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_SET_KEYSPACE_TIMEOUT)) .thenReturn(Duration.ofMillis(TIMEOUT_MILLIS)); when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_REQUESTS)).thenReturn(1); + // The reference.conf default; individual tests may lower it to exercise the cap. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(5); when(defaultProfile.getDuration(DefaultDriverOption.HEARTBEAT_INTERVAL)) .thenReturn(Duration.ofSeconds(30)); when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT)) @@ -131,6 +152,8 @@ public void setup() throws InterruptedException { when(context.getProtocolVersionRegistry()).thenReturn(protocolVersionRegistry); when(context.getNettyOptions()).thenReturn(nettyOptions); when(nettyOptions.ioEventLoopGroup()).thenReturn(clientGroup); + when(nettyOptions.adminEventExecutorGroup()).thenReturn(adminGroup); + when(nettyOptions.getTimer()).thenReturn(timer); when(nettyOptions.channelClass()).thenAnswer((Answer) i -> LocalChannel.class); when(nettyOptions.allocator()).thenReturn(ByteBufAllocator.DEFAULT); when(context.getFrameCodec()) @@ -188,6 +211,55 @@ protected Frame readOutboundFrame() { return null; // never reached } + /** + * Like {@link #readOutboundFrame()}, but returns {@code null} instead of failing the test when no + * frame arrives within {@code timeoutMillis}. + * + *

Use this to assert that the client did not send another request. Unlike asserting via + * a failing read, it also drains a frame that does arrive: the server-side exchange in {@link + * ServerInitializer} has no timeout, so a stray unread frame would block the server event loop + * and hang the whole suite in {@link #tearDown()} instead of failing just the test. + */ + protected Frame tryReadOutboundFrame(long timeoutMillis) { + try { + return requestFrameExchanger.exchange(null, timeoutMillis, MILLISECONDS); + } catch (InterruptedException e) { + fail("unexpected interruption while waiting for outbound frame", e); + return null; // never reached + } catch (TimeoutException e) { + return null; + } + } + + /** + * A {@link Random} that turns {@link java.util.Collections#shuffle} into a no-op, so a test whose + * scenario depends on which address is tried first can rely on the order the resolver returned: + * shuffle swaps element {@code i-1} with {@code nextInt(i)}, and returning {@code i-1} swaps + * every element with itself. + * + *

Assign it to {@link ChannelFactory#random}, the injection point that exists for this. + */ + static final class KeepResolverOrder extends Random { + private static final long serialVersionUID = 1L; + + @Override + public int nextInt(int bound) { + return bound - 1; + } + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + protected void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + } + protected void writeInboundFrame(Frame requestFrame, Message response) { writeInboundFrame(requestFrame, response, requestFrame.protocolVersion); } @@ -295,5 +367,7 @@ public void tearDown() throws InterruptedException { clientGroup .shutdownGracefully(TIMEOUT_MILLIS, TIMEOUT_MILLIS * 2, TimeUnit.MILLISECONDS) .sync(); + adminGroup.shutdownGracefully(TIMEOUT_MILLIS, TIMEOUT_MILLIS * 2, TimeUnit.MILLISECONDS).sync(); + timer.stop(); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java index 682caac198d..322af30dc84 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java @@ -36,7 +36,6 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; -import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.DefaultProtocolVersionRegistry; import com.datastax.oss.driver.internal.core.ProtocolVersionRegistry; @@ -50,11 +49,9 @@ import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; -import com.datastax.oss.protocol.internal.ProtocolConstants.ErrorCode; import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,6 +156,52 @@ public void should_initialize() { assertThat(connectFuture).isSuccess(); } + /** + * Drives initialization as far as the cluster name check, which is where node identification + * branches off. + * + * @return the connect future, so that the caller can assert on its outcome. + */ + private ChannelFuture initUpToClusterName(DriverChannelOptions options) { + channel + .pipeline() + .addLast( + ChannelFactory.INIT_HANDLER_NAME, + new ProtocolInitHandler( + internalDriverContext, + DefaultProtocolVersion.V4, + null, + END_POINT, + options, + heartbeatHandler, + false)); + + ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(buildInboundFrame(requestFrame, new Ready())); + + requestFrame = readOutboundFrame(); + assertThat(((Query) requestFrame.message).query) + .isEqualTo("SELECT cluster_name FROM system.local WHERE key='local'"); + writeInboundFrame(requestFrame, TestResponses.clusterNameResponse("someClusterName")); + + return connectFuture; + } + + @Test + public void should_complete_init_after_cluster_name_check() { + // Given — no keyspace to set. The node-identity read and the REGISTER request both happen + // after initialization (through the connect hook and ChannelFactory respectively), so init + // itself ends at the cluster-name check: byte for byte the pre-multi-address exchange. + ChannelFuture connectFuture = initUpToClusterName(DriverChannelOptions.DEFAULT); + + // Then + assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); + } + // Mirrors the real reporter, which only ever sees the control connection. private void stubConfigReporter() { when(internalDriverContext.getDriverConfigReporter()) @@ -601,7 +644,10 @@ public void should_initialize_with_keyspace() { } @Test - public void should_initialize_with_events() { + public void should_not_send_register_during_init_even_when_events_are_requested() { + // REGISTER moved out of initialization: ChannelFactory sends it after the connect hook has + // accepted the channel, so a candidate about to be rejected never registers for events. Init + // itself must therefore end without a Register frame even when the options ask for events. List eventTypes = ImmutableList.of("foo", "bar"); EventCallback eventCallback = mock(EventCallback.class); DriverChannelOptions driverChannelOptions = @@ -624,12 +670,8 @@ public void should_initialize_with_events() { writeInboundFrame(readOutboundFrame(), new Ready()); writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - Frame requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -664,12 +706,10 @@ public void should_initialize_with_keyspace_and_events() { assertThat(((Query) requestFrame.message).query).isEqualTo("USE \"ks\""); writeInboundFrame(requestFrame, new SetKeyspace("ks")); - requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - + // No Register frame: setting the keyspace is now the last init step, events are registered by + // ChannelFactory after the connect hook. assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -745,56 +785,4 @@ public void should_fail_pending_requests_only_once_if_init_fails() { logger.detachAppender(appender); logger.setLevel(levelBefore); } - - @Test - public void should_fail_connection_when_client_routes_change_rejected() { - List eventTypes = - ImmutableList.of( - ProtocolConstants.EventType.SCHEMA_CHANGE, - ProtocolConstants.EventType.STATUS_CHANGE, - ProtocolConstants.EventType.TOPOLOGY_CHANGE, - ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - EventCallback eventCallback = mock(EventCallback.class); - DriverChannelOptions driverChannelOptions = - DriverChannelOptions.builder().withEvents(eventTypes, eventCallback).build(); - channel - .pipeline() - .addLast( - ChannelFactory.INIT_HANDLER_NAME, - new ProtocolInitHandler( - internalDriverContext, - DefaultProtocolVersion.V4, - null, - END_POINT, - driverChannelOptions, - heartbeatHandler, - false)); - - ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); - - // STARTUP - writeInboundFrame(readOutboundFrame(), new Ready()); - // Cluster name check - writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - - // REGISTER attempt includes CLIENT_ROUTES_CHANGE - Frame registerFrame = readOutboundFrame(); - assertThat(registerFrame.message).isInstanceOf(Register.class); - Register firstRegister = (Register) registerFrame.message; - assertThat(firstRegister.eventTypes).contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - - // Server rejects with PROTOCOL_ERROR mentioning CLIENT_ROUTES_CHANGE - writeInboundFrame( - registerFrame, - new Error( - ErrorCode.PROTOCOL_ERROR, - "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); - - // Connection must fail with a clear error message - assertThat(connectFuture).isFailed(); - assertThat(connectFuture.cause()) - .isInstanceOf(ConnectionInitException.class) - .hasMessageContaining("CLIENT_ROUTES_CHANGE") - .hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); - } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java new file mode 100644 index 00000000000..1caaa58c7e8 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code + * DnsAddressResolverGroup}), installed the way a user would install one: through {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Records what it was asked to resolve, and answers with a fixed list of addresses so tests can + * assert that every one of them is tried, and in what order. + * + *

Implements {@link AddressResolver} directly rather than extending {@code + * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect over + * Netty's local transport, which is not reachable through an {@link InetSocketAddress}. + */ +class TestAddressResolverGroup extends AddressResolverGroup { + + /** Every address this group was asked to resolve, in order. */ + final List queried = new CopyOnWriteArrayList<>(); + + /** Whether a resolver was ever obtained from this group at all. */ + volatile boolean resolverRequested; + + /** The executor the last resolver was created for, i.e. the loop resolution runs on. */ + @Nullable volatile EventExecutor resolverExecutor; + + /** The addresses to answer with, or {@code null} to fail every lookup. */ + @Nullable private final List answer; + + /** + * Whether to claim that every address still needs resolving, even one that already carries an IP. + * A real resolver may do this to redirect traffic, and Netty honours it: {@code + * Bootstrap#doResolveAndConnect0} asks the resolver rather than testing the address itself. + */ + private final boolean claimNothingIsResolved; + + /** + * Whether to decline every address, as a real resolver does for an address type it does not + * handle ({@code DefaultNameResolver} declines anything that is not an {@link + * InetSocketAddress}). Netty then passes the address through untouched, and so does the driver. + */ + private final boolean declineEverything; + + /** + * Whether to claim that every address is already resolved, even one that is plainly a name. That + * is what {@code NoopAddressResolverGroup} does — Netty's way of saying "leave the name alone, + * something in the pipeline will deal with it", which is how a {@code ProxyHandler} gets the name + * rather than an IP. Netty hands such an address straight to {@code doConnect()}. + */ + private final boolean claimEverythingIsResolved; + + /** A stand-in for {@code NoopAddressResolverGroup}: supports everything, resolves nothing. */ + static TestAddressResolverGroup claimingEverythingIsResolved() { + return new TestAddressResolverGroup(null, false, false, true); + } + + TestAddressResolverGroup(@Nullable List answer) { + this(answer, false); + } + + TestAddressResolverGroup(@Nullable List answer, boolean claimNothingIsResolved) { + this(answer, claimNothingIsResolved, false); + } + + TestAddressResolverGroup( + @Nullable List answer, + boolean claimNothingIsResolved, + boolean declineEverything) { + this(answer, claimNothingIsResolved, declineEverything, false); + } + + private TestAddressResolverGroup( + @Nullable List answer, + boolean claimNothingIsResolved, + boolean declineEverything, + boolean claimEverythingIsResolved) { + this.answer = answer; + this.claimNothingIsResolved = claimNothingIsResolved; + this.declineEverything = declineEverything; + this.claimEverythingIsResolved = claimEverythingIsResolved; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + resolverRequested = true; + resolverExecutor = executor; + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + return !declineEverything; + } + + @Override + public boolean isResolved(SocketAddress address) { + if (claimEverythingIsResolved) { + return true; + } + if (claimNothingIsResolved) { + return false; + } + // Only hostnames need resolving; anything else (including the local-transport addresses we + // hand back) is already usable. + return !(address instanceof InetSocketAddress) + || !((InetSocketAddress) address).isUnresolved(); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve(SocketAddress address, Promise promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer.get(0)); + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer); + } + + @Override + public void close() { + // nothing to do + } + }; + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java index d51ee445da5..7178d40b389 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java @@ -21,37 +21,68 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.channel.ChannelEvent; +import com.datastax.oss.driver.internal.core.channel.ChannelFactory; +import com.datastax.oss.driver.internal.core.channel.ConnectHook; import com.datastax.oss.driver.internal.core.channel.DriverChannel; +import com.datastax.oss.driver.internal.core.channel.DriverChannelOptions; import com.datastax.oss.driver.internal.core.channel.MockChannelFactoryHelper; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.DefaultNodeInfo; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; import com.datastax.oss.driver.internal.core.metadata.TestNodeFactory; import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import java.net.ConnectException; +import java.net.InetSocketAddress; import java.time.Duration; +import java.util.AbstractMap.SimpleEntry; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.slf4j.LoggerFactory; @RunWith(DataProviderRunner.class) public class ControlConnectionTest extends ControlConnectionTestBase { + @Mock private Appender appender; + @Test public void should_close_successfully_if_it_was_never_init() { // When @@ -61,6 +92,268 @@ public void should_close_successfully_if_it_was_never_init() { assertThatStage(closeFuture).isSuccess(); } + @Test + public void should_arm_connect_hook_for_a_node_whose_host_id_is_unknown() { + // Given — a contact point: DefaultNode.newContactPoint leaves hostId null, and that is the one + // case with something to learn. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then — the identity read happens through the connect hook, inside the factory's candidate + // loop, where an address that cannot identify itself is rejected while the hostname's other + // addresses are still on hand. + DriverChannelOptions options = connectOptions(contactPoint); + assertThat(options.connectHook).isNotNull(); + assertThat(options.connectHookTimeout).isNotNull(); + } + + @Test + public void should_reject_candidate_when_hook_reads_no_host_id() { + // Given — the hook armed for a contact point, and a topology monitor that answers the identity + // read with a NodeInfo carrying no host id (a bootstrapping node). + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + NodeInfo anonymousInfo = mock(NodeInfo.class); + when(anonymousInfo.getHostId()).thenReturn(null); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(anonymousInfo)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the factory invokes the hook against the candidate channel. + CompletionStage vetted = connectOptions(contactPoint).connectHook.onConnect(channel); + + // Then — the candidate is rejected, so the factory can move on to the next address; accepting + // it would fail later in registerNode and cost the whole plan entry. + assertThatStage(vetted).isFailed(error -> assertThat(error).hasMessageContaining("host id")); + } + + @Test + public void should_read_each_candidates_columns_from_scratch() { + // The identity read teaches DefaultTopologyMonitor the system.local column projection, and + // until this hook existed it only ever ran against the channel the control connection kept -- + // so what it learned was by construction the accepted node's. It now runs once per candidate + // address, and the projection is an intersection: it can only shrink. Whichever candidate the + // loop ends up keeping, the projection has to be that node's. + // + // Which is why the cache is cleared *before* the read rather than undone after a rejection. + // Undoing it on the rejection paths cannot work: none of them sees what a previous candidate + // left behind, so a first address refused after its read would still narrow what a second, + // accepted one reports -- and two of those paths (a REGISTER rejection, a hook abandoned on + // its timeout) are ChannelFactory's, invisible from here. Clearing first makes every read a + // SELECT * that re-learns from whoever answered. + // + // The ordering is the assertion. A reset after the read would leave no projection at all. + // Which candidate's answer survives is a separate question -- the reads are ordered, the + // *responses* are not -- and the two tests below cover two of the three ways they can + // interleave. See DefaultTopologyMonitor#toLocalNodeInfo for the third, which is deferred. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + // Hoisted: calling a mock inside when(...) leaves Mockito with an unfinished stubbing. + EndPoint channelEndPoint = channel.getEndPoint(); + NodeInfo nodeInfo = mock(NodeInfo.class); + when(nodeInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(nodeInfo.getEndPoint()).thenReturn(channelEndPoint); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(nodeInfo)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- the factory invokes the hook against the candidate channel. + assertThatStage(connectOptions(contactPoint).connectHook.onConnect(channel)).isSuccess(); + + // Then + InOrder inOrder = inOrder(topologyMonitor); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(channel); + // And only the local projection. The hook reads system.local, so that is the only cache it can + // have narrowed; discarding the peer projections too would cost a SELECT * over every peer row + // on the next refresh, to recover from one address of one contact point. + verify(topologyMonitor, never()).resetColumnCaches(); + } + + @Test + public void should_not_let_a_refused_candidate_narrow_the_next_ones_projection() { + // Two addresses of one contact point: the first identifies nobody and is refused, the second is + // accepted. Both reads must start from a cleared cache, or the refused node's columns are what + // the session goes on using -- it read first, and the intersection only shrinks. This is the + // case a per-rejection undo cannot reach, because the rejection that matters happens before the + // read that would have to be corrected. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel refused = newMockDriverChannel(3); + DriverChannel accepted = newMockDriverChannel(4); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, accepted).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + EndPoint acceptedEndPoint = accepted.getEndPoint(); + NodeInfo anonymousInfo = mock(NodeInfo.class); + when(anonymousInfo.getHostId()).thenReturn(null); + when(topologyMonitor.getChannelNodeInfo(refused)) + .thenReturn(CompletableFuture.completedFuture(anonymousInfo)); + NodeInfo acceptedInfo = mock(NodeInfo.class); + when(acceptedInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(acceptedInfo.getEndPoint()).thenReturn(acceptedEndPoint); + when(topologyMonitor.getChannelNodeInfo(accepted)) + .thenReturn(CompletableFuture.completedFuture(acceptedInfo)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + ConnectHook hook = connectOptions(contactPoint).connectHook; + + // When -- the factory walks the two candidates through the one hook it was given. + assertThatStage(hook.onConnect(refused)) + .isFailed(error -> assertThat(error).hasMessageContaining("host id")); + assertThatStage(hook.onConnect(accepted)).isSuccess(); + + // Then -- a clear before each read, so the accepted node's answer is the one left in the cache. + InOrder inOrder = inOrder(topologyMonitor); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(refused); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(accepted); + verify(topologyMonitor, never()).resetColumnCaches(); + } + + @Test + public void should_not_arm_connect_hook_for_a_node_that_is_already_identified() { + // Given — node1 has a host id, so there is nothing to learn and nothing to vet: its + // connections keep the exact exchange they had before. + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + + // Then + assertThat(connectOptions(node1).connectHook).isNull(); + } + + @Test + public void should_use_node_info_captured_by_hook_without_reading_again() { + // Given — a contact point whose connect hook has run against the winning channel, capturing + // the identity it read. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + CompletableFuture channelFuture = new CompletableFuture<>(); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .pending(contactPoint, channelFuture) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + EndPoint channelEndPoint = channel.getEndPoint(); + NodeInfo nodeInfo = mock(NodeInfo.class); + when(nodeInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(nodeInfo.getEndPoint()).thenReturn(channelEndPoint); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(nodeInfo)); + DefaultNode registered = TestNodeFactory.newNode(3, context); + when(metadataManager.registerNode(nodeInfo)) + .thenReturn(CompletableFuture.completedFuture(registered)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the hook runs once (as the factory would against the accepted candidate), then the + // connect completes with that same channel. + assertThatStage(connectOptions(contactPoint).connectHook.onConnect(channel)).isSuccess(); + channelFuture.complete(channel); + + // Then — what the hook captured is what gets registered, with no second identity read: the + // monitor was asked exactly once, by the hook itself. + verify(metadataManager, VERIFY_TIMEOUT).registerNode(nodeInfo); + verify(topologyMonitor, times(1)).getChannelNodeInfo(channel); + } + + @Test + public void should_clear_the_projection_before_falling_back_to_a_direct_read() { + // The other interleaving. A candidate abandoned on the hook timeout is not cancelled, so its + // system.local answer can land *after* the accepted candidate's -- and then it is that late + // write the holder carries, the projection in the cache is the refused node's, and clearing + // before the next read cannot help because there is no next hook read. What saves it is the + // same late write: pairing the capture with its channel makes getFor miss, and the fallback + // read that follows has to start from a cleared cache like any other. Without the clear, the + // node the driver is keeping is identified through a projection intersected against one it + // refused, which is the whole failure the reset exists to prevent. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel accepted = newMockDriverChannel(3); + DriverChannel stranded = newMockDriverChannel(4); + CompletableFuture channelFuture = new CompletableFuture<>(); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .pending(contactPoint, channelFuture) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + EndPoint acceptedEndPoint = accepted.getEndPoint(); + NodeInfo acceptedInfo = mock(NodeInfo.class); + when(acceptedInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(acceptedInfo.getEndPoint()).thenReturn(acceptedEndPoint); + when(topologyMonitor.getChannelNodeInfo(accepted)) + .thenReturn(CompletableFuture.completedFuture(acceptedInfo)); + EndPoint strandedEndPoint = stranded.getEndPoint(); + NodeInfo strandedInfo = mock(NodeInfo.class); + when(strandedInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(strandedInfo.getEndPoint()).thenReturn(strandedEndPoint); + when(topologyMonitor.getChannelNodeInfo(stranded)) + .thenReturn(CompletableFuture.completedFuture(strandedInfo)); + DefaultNode registered = TestNodeFactory.newNode(3, context); + when(metadataManager.registerNode(acceptedInfo)) + .thenReturn(CompletableFuture.completedFuture(registered)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + ConnectHook hook = connectOptions(contactPoint).connectHook; + + // When — the hook runs against the candidate the loop keeps, and then the abandoned one + // answers, overwriting the capture. The connect completes with the accepted channel. + assertThatStage(hook.onConnect(accepted)).isSuccess(); + assertThatStage(hook.onConnect(stranded)).isSuccess(); + channelFuture.complete(accepted); + + // Then -- the stale capture is not used, and the read that replaces it is preceded by its own + // clear. The last of the three resets is the one this test is about: drop it and the fallback + // read reuses whatever the stranded candidate taught the monitor. + InOrder inOrder = inOrder(topologyMonitor); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(accepted); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(stranded); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).resetLocalColumnCache(); + inOrder.verify(topologyMonitor, VERIFY_TIMEOUT).getChannelNodeInfo(accepted); + verify(metadataManager, VERIFY_TIMEOUT).registerNode(acceptedInfo); + verify(metadataManager, never()).registerNode(strandedInfo); + verify(topologyMonitor, never()).resetColumnCaches(); + } + + /** The options {@code ChannelFactory} was asked to connect {@code node} with. */ + private DriverChannelOptions connectOptions(Node node) { + ArgumentCaptor captor = + ArgumentCaptor.forClass(DriverChannelOptions.class); + verify(channelFactory, VERIFY_TIMEOUT).connect(eq(node), captor.capture()); + return captor.getValue(); + } + @Test public void should_init_with_first_contact_point_if_reachable() { // Given @@ -98,6 +391,39 @@ public void should_always_return_same_init_future() { factoryHelper.verifyNoMoreCalls(); } + @Test + public void should_advance_to_next_node_when_a_listener_throws_on_connection_failure() { + // EventBus.fire() has no try/catch of its own, and RunOrSchedule.on(adminExecutor, ..) runs + // listeners inline when the firing thread is already the admin loop -- so a NodeStateListener + // or metrics chain that throws propagates out of the fire() call. It sits immediately before + // the recursive connect() that advances the query plan, inside an outer catch that only logs, + // so without the guard the round neither advances nor completes: initFuture stays pending + // forever (SessionBuilder.build() blocks on it) and no failure is ever reported. + // + // Modelled by making the bus itself throw at that call, which is what an inline listener does. + DriverChannel channel2 = newMockDriverChannel(2); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .failure(node1, "mock failure") + .success(node2, channel2) + .build(); + doThrow(new IllegalStateException("listener blew up")) + .when(eventBus) + .fire(ChannelEvent.controlConnectionFailed(node1)); + + // When + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + factoryHelper.waitForCall(node2); + + // Then the listener's failure is absorbed and the plan still advances. + assertThatStage(initFuture) + .isSuccess(v -> assertThat(controlConnection.channel()).isEqualTo(channel2)); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(node2)); + + factoryHelper.verifyNoMoreCalls(); + } + @Test public void should_init_with_second_contact_point_if_first_one_fails() { // Given @@ -819,149 +1145,1309 @@ public void should_resolve_contact_point_to_existing_metadata_node_on_reconnect( } @Test - public void should_return_control_node_after_init() { - // Given + public void should_keep_the_control_channel_pinned_when_it_adopts_a_nodes_endpoint() { + // Given — a Cloud/SNI contact point. The channel's endpoint is pinned to the one proxy IP the + // connection reached; the node's own endpoint is the unpinned form, which resolves to the + // *unresolved* proxy address that every SniEndPoint in the cluster shares. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + EndPoint unpinned = new SniEndPoint(proxy, node2.getHostId().toString()); + InetSocketAddress reachedProxyIp = new InetSocketAddress("10.0.0.5", 9042); + EndPoint pinned = ((SniEndPoint) unpinned).pinTo(reachedProxyIp); + + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); DriverChannel channel1 = newMockDriverChannel(1); + when(channel1.getEndPoint()).thenReturn(pinned); + mockQueryPlan(contactPoint); MockChannelFactoryHelper factoryHelper = - MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel1).build(); - // Before init, controlNode should be null - assertThat(controlConnection.controlNode()).isNull(); + NodeInfo resolvedInfo = + DefaultNodeInfo.builder().withEndPoint(unpinned).withHostId(node2.getHostId()).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel1)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())).thenReturn(CompletableFuture.completedFuture(node2)); // When CompletionStage initFuture = controlConnection.init(false, false, false); - factoryHelper.waitForCall(node1); - - // Then — after init, controlNode should be set + factoryHelper.waitForCall(contactPoint); assertThatStage(initFuture).isSuccess(); - await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); - assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node1.getHostId()); + + // Then — the channel adopts the node's identity but keeps the address it is actually connected + // to. Adopting the unpinned form would make channel.getEndPoint().resolve() the shared proxy + // name, which every other node's endpoint equals -- so isControlNode()'s resolve() comparison + // would answer true for any node in the cluster during the resolution window, and JAVA-2303's + // self-peer guard (broadcastRpcAddress.equals(localEndPoint.resolve())) would stop matching, + // an unresolved address never equalling a resolved one. + ArgumentCaptor adopted = ArgumentCaptor.forClass(EndPoint.class); + verify(channel1, VERIFY_TIMEOUT).setEndPoint(adopted.capture()); + assertThat(adopted.getValue().resolve()).isEqualTo(reachedProxyIp); + // Still the node's endpoint by identity: only the pin differs. + assertThat(adopted.getValue()).isEqualTo(unpinned); + assertThat(adopted.getValue().asMetricPrefix()).isEqualTo(unpinned.asMetricPrefix()); factoryHelper.verifyNoMoreCalls(); } @Test - public void should_clear_control_node_on_channel_close_and_restore_after_reconnect() { - // Given + public void should_refuse_a_contact_point_that_resolves_to_a_removed_node() { + // Given — node2 has been removed by the topology monitor, and a contact point whose DNS record + // still lists it. This is what the reconnection fallback makes reachable: the plan is + // exhausted, + // the contact points are re-offered, and one of them leads back to a node the cluster no longer + // has. Registering it would publish it back into Metadata#getNodes() and fire onAdd for it, and + // the driver would then keep its control connection on a decommissioned node -- whose own + // system.peers view is what the next refresh would adopt. when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); DriverChannel channel1 = newMockDriverChannel(1); DriverChannel channel2 = newMockDriverChannel(2); - CompletableFuture channel2Future = new CompletableFuture<>(); + DriverChannel channel3 = newMockDriverChannel(3); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); MockChannelFactoryHelper factoryHelper = MockChannelFactoryHelper.builder(channelFactory) - .success(node1, channel1) - .failure(node1, "mock failure") - .pending(node2, channel2Future) + .success(node1, channel1) // init + .success(contactPoint, channel2) // reconnect via the fallback: reaches removed node2 + .success(node1, channel3) // ... and moves on to the next plan entry .build(); CompletionStage initFuture = controlConnection.init(false, false, false); factoryHelper.waitForCall(node1); assertThatStage(initFuture).isSuccess(); - await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); - assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node1.getHostId()); - // When — channel closes, reconnection starts but pending on node2 - channel1.close(); - verify(reconnectionSchedule, VERIFY_TIMEOUT).nextDelay(); - factoryHelper.waitForCall(node1); // fails - factoryHelper.waitForCall(node2); // pending + // node2 is removed: the event names the instance, and it leaves metadata. + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(node2.getHostId()); - // Then — controlNode should be null during reconnection window - assertThat(controlConnection.controlNode()).isNull(); + NodeInfo removedNodeInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(node2.getHostId()) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(removedNodeInfo)); - // Complete the pending reconnection - channel2Future.complete(channel2); + // When + mockQueryPlan(contactPoint, node1); + channel1.close(); - // After reconnection, controlNode should be set to node2 - await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel2)); - await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); - assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node2.getHostId()); + // Then — the channel to the removed node is closed and the plan advances. Crucially the node is + // never registered: the instance-keyed guards cannot answer for a node registerNode would mint + // fresh, so the decision has to be taken on the host id, before registration. + factoryHelper.waitForCall(contactPoint); + verify(channel2, VERIFY_TIMEOUT).forceClose(); + factoryHelper.waitForCall(node1); + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel3)); + + verify(metadataManager, never()).registerNode(any()); + assertThat(registeredNodes).doesNotContainKey(node2.getHostId()); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(node2)); factoryHelper.verifyNoMoreCalls(); } @Test - public void should_not_reconnect_on_event_for_non_control_node() { - // Given — init with node1 (has hostId) + public void should_not_count_a_wholly_excluded_contact_point_as_a_connection_failure() { + // Given — a contact point every one of whose addresses the connect hook refused on its host id. + // ChannelFactory reports that as a single failure for the name, and it arrives wrapped twice: + // the hook's stage fails with a CompletionException, and finishCandidate() puts a + // ConnectionInitException around that before the candidate loop sees it. Matching on one fixed + // shape would read this as a node that could not be reached. + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel3 = newMockDriverChannel(3); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + Throwable asChannelFactoryReportsIt = + new ConnectionInitException( + "Connect hook rejected the channel", + new CompletionException( + new ControlConnection.ExcludedNodeException("node was removed"))); MockChannelFactoryHelper factoryHelper = - MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .failure(contactPoint, asChannelFactoryReportsIt) // reconnect via the fallback + .success(node1, channel3) + .build(); CompletionStage initFuture = controlConnection.init(false, false, false); factoryHelper.waitForCall(node1); assertThatStage(initFuture).isSuccess(); - await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); - // When — fire a distance event for node2 (not the control node) - eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, node2)); + // When + mockQueryPlan(contactPoint, node1); + channel1.close(); - // Then — should NOT trigger reconnection (channel stays the same) - await() - .during(Duration.ofMillis(200)) - .untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel1)); + // Then — the plan advances, but nothing failed to connect, so no controlConnectionFailed event + // is fired for the contact point. Firing one would count a refusal as a connection failure in + // the node metrics and warn the operator about a deployment that is in fact reachable. + factoryHelper.waitForCall(contactPoint); + factoryHelper.waitForCall(node1); + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel3)); + verify(eventBus, never()).fire(ChannelEvent.controlConnectionFailed(contactPoint)); factoryHelper.verifyNoMoreCalls(); } @Test - public void should_reconnect_when_control_node_removed_from_metadata_after_reconnect() { - // Given — init with node1 + public void should_refuse_a_removed_host_id_inside_the_connect_hook() { + // Given — the same situation as above, but looked at one address earlier. A contact point is a + // *name*: ChannelFactory expands it and tries the addresses in turn, and only the first one to + // be accepted becomes the connection. Deciding the host id after that -- which is where + // #resolveChannelNodeIfNeeded still decides it, on live state -- is a decision taken when the + // other candidates have already been discarded, so a name whose first shuffled address happens + // to be the removed node writes off every other address behind it for that round. when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); DriverChannel channel1 = newMockDriverChannel(1); DriverChannel channel2 = newMockDriverChannel(2); DriverChannel channel3 = newMockDriverChannel(3); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); MockChannelFactoryHelper factoryHelper = MockChannelFactoryHelper.builder(channelFactory) .success(node1, channel1) // init - .success(node2, channel2) // first reconnect - .failure(node2, "decommissioned") // second reconnect: node2 fails - .success(node1, channel3) // second reconnect: falls back to node1 + .success(contactPoint, channel2) // reconnect via the fallback + .success(node1, channel3) .build(); CompletionStage initFuture = controlConnection.init(false, false, false); factoryHelper.waitForCall(node1); assertThatStage(initFuture).isSuccess(); - // Remove node2 from metadata before triggering reconnect - // (simulating node2 was decommissioned during the outage) + eventBus.fire(NodeStateEvent.removed(node2)); registeredNodes.remove(node2.getHostId()); - // When — channel1 goes down, reconnects to node2 - mockQueryPlan(node2, node1); + NodeInfo removedNodeInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(node2.getHostId()) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(removedNodeInfo)); + + mockQueryPlan(contactPoint, node1); channel1.close(); + factoryHelper.waitForCall(contactPoint); - // Then — onSuccessfulReconnect detects node2 is gone → triggers second reconnection - // which eventually connects to node1 (channel3) - factoryHelper.waitForCall(node2); // first reconnect + // When — the factory runs the hook against a candidate that turns out to be the removed node. + CompletionStage vetted = connectOptions(contactPoint).connectHook.onConnect(channel2); - // onSuccessfulReconnect detects node2 is removed from metadata → force-closes channel2 - verify(channel2, VERIFY_TIMEOUT).forceClose(); + // Then — that one candidate is rejected, and with a cause the factory does not treat as + // node-wide, so the loop moves on to the hostname's remaining addresses. + assertThatStage(vetted) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ControlConnection.ExcludedNodeException.class); + assertThat(error).hasMessageContaining("node was removed"); + }); + } - factoryHelper.waitForCall(node2); // second reconnect: node2 fails - factoryHelper.waitForCall(node1); // second reconnect: node1 succeeds - await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel3)); - // channelOpened(node1) fires twice: once during init, once during second reconnect - verify(eventBus, timeout(500).times(2)).fire(ChannelEvent.channelOpened(node1)); + @Test + public void should_accept_a_host_id_nothing_is_known_against_inside_the_connect_hook() { + // The other side of the same gate, and the reason it cannot simply refuse anything unfamiliar: + // a host id in neither metadata nor the removed set is a node that joined while the driver was + // disconnected, which is exactly what the contact-point fallback exists to reach. + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) + .success(contactPoint, channel2) + .build(); - factoryHelper.verifyNoMoreCalls(); + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(node2.getHostId()); + + NodeInfo newcomerInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(UUID.randomUUID()) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(newcomerInfo)); + + mockQueryPlan(contactPoint, node1); + channel1.close(); + factoryHelper.waitForCall(contactPoint); + + assertThatStage(connectOptions(contactPoint).connectHook.onConnect(channel2)).isSuccess(); } @Test - public void should_not_call_getChannelNodeInfo_for_metadata_node_with_hostId() { - // Given — node1 already has a hostId (it's a metadata node, not a contact point) + public void should_stop_refusing_removed_nodes_once_a_whole_reconnection_round_has_failed() { + // The removal set is only ever cleared by a node state event, and those arrive from a metadata + // refresh -- which needs the very control connection this is trying to restore. So a host id + // recorded as removed on stale information (a node transiently missing from a peers table) + // would otherwise refuse the only reachable address for the rest of the session, and the + // contact-point fallback could never recover from it. A round that reaches nothing at all is + // the signal that this knowledge is no longer worth acting on. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); MockChannelFactoryHelper factoryHelper = - MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .failure(contactPoint, "mock failure") // a round that reaches nothing + .success(contactPoint, channel2) // the next round, no longer refused + .build(); - // When CompletionStage initFuture = controlConnection.init(false, false, false); factoryHelper.waitForCall(node1); - - // Then — resolveChannelNodeIfNeeded should short-circuit assertThatStage(initFuture).isSuccess(); + + // node2 is removed, and never reports a state again -- nothing can un-remove it the usual way. + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + NodeInfo removedNodeInfo = + DefaultNodeInfo.builder().withEndPoint(channel2.getEndPoint()).withHostId(hostId).build(); TopologyMonitor topologyMonitor = context.getTopologyMonitor(); - verify(topologyMonitor, never()).getChannelNodeInfo(any(DriverChannel.class)); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(removedNodeInfo)); + // As the real MetadataManager does for a host id absent from metadata: mint a fresh node and + // publish it. Both matter here -- returning the *same* instance would keep it in the + // instance-keyed lastNodeState map and hit the separate post-registration guard, and not + // publishing it would have the control connection drop the node again for being absent and + // reconnect in a loop. Either would be a property of the stub, not of the code under test. + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); - factoryHelper.verifyNoMoreCalls(); + // When -- the contact point is all that is left, and the first round against it fails outright. + mockQueryPlan(contactPoint); + channel1.close(); + + // Then -- the round that follows reaches the same host id and is allowed through this time, + // rather than being refused forever on the strength of a removal the driver can no longer + // confirm or retract. + factoryHelper.waitForCalls(contactPoint, 2); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(reRegistered)); + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel2)); + } + + @Test + public void should_keep_refusing_a_removed_node_while_the_refusal_is_within_its_budget() { + // The counterpart to the test above. That clear exists for a round that could reach nothing, at + // which point the driver's view of who was removed is as stale as its view of everything else. + // A round drained purely by refusals is not that: every node in it answered, and the refusal is + // this very set doing its job. Clearing on it would undo the protection on the round that just + // enforced it, handing the next round the node it had only just turned away. + // + // The refusal is budgeted rather than unconditional (see the test below), so the schedule here + // hands out two quick delays and then a very long one: exactly two rounds run, both inside the + // budget, and nothing races the assertions. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()) + .thenReturn(Duration.ofNanos(1), Duration.ofNanos(1), Duration.ofDays(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .success(contactPoint, channel2) // round 1: connects, then is refused by host id + .success(contactPoint, channel3) // round 2: refused again, the set having survived + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + // Both rounds reach the same removed host id: the contact point's DNS still lists it. Built + // before the stubbing, since getEndPoint() is itself a mock call. A fresh channel per round, + // because forceClose() completes the mock's close future and a reused one would come back as + // "channel closed during endpoint resolve" -- a connectivity failure, which clears the set for + // an entirely different reason and would make this test pass without testing anything. + NodeInfo round1Info = + DefaultNodeInfo.builder().withEndPoint(channel2.getEndPoint()).withHostId(hostId).build(); + NodeInfo round2Info = + DefaultNodeInfo.builder().withEndPoint(channel3.getEndPoint()).withHostId(hostId).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(round1Info)); + when(topologyMonitor.getChannelNodeInfo(channel3)) + .thenReturn(CompletableFuture.completedFuture(round2Info)); + // Stubbed so that a round which wrongly stopped refusing would go on to succeed rather than + // NPE: without the gate this is what round 2 reaches, and the assertions below would then be + // failing on the behaviour rather than on the stub. + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); + + mockQueryPlan(contactPoint); + channel1.close(); + + factoryHelper.waitForCalls(contactPoint, 2); + // Waited for on a terminal signal rather than asserted on the spot: connect() parks the new + // channel on the control connection *before* the identity read and only nulls it once the host + // id comes back refused, so controlConnection.channel() is transiently the refused channel. + // Closing it is the refusal itself, and cannot be observed early. + verify(channel3, VERIFY_TIMEOUT).forceClose(); + // Neither round got as far as adopting the node: the host id is checked before registerNode, + // so a refusal leaves the driver's view of it untouched. + verify(eventBus, never()).fire(ChannelEvent.channelOpened(reRegistered)); + assertThat(registeredNodes).doesNotContainKey(hostId); + } + + @Test + public void should_keep_refusing_a_removed_node_when_a_sibling_address_was_unreachable() { + // A contact point whose addresses went [refused, unreachable]. ChannelFactory reports one + // failure for the name and chooses which half speaks, so this is the ordering where the + // refusal surfaces and the connect failure rides along as suppressed -- the other ordering is + // the same round seen from the other side. + // + // That round must not discard the removal set. Clearing exists for a round that could reach + // nothing, at which point the driver's view of who was removed is as stale as its view of + // everything else; this round handshaked with a live server and read its host id, which is the + // opposite of stale. Letting one firewalled sibling record undo the refusal its neighbour just + // earned is how the removed node comes back on the very next round. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()) + .thenReturn(Duration.ofNanos(1), Duration.ofNanos(1), Duration.ofDays(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel3 = newMockDriverChannel(3); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + Throwable refusedThenUnreachable = + new ConnectionInitException( + "Connect hook rejected the channel", + new CompletionException( + new ControlConnection.ExcludedNodeException("node was removed"))); + refusedThenUnreachable.addSuppressed(new ConnectException("connection refused")); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .failure(contactPoint, refusedThenUnreachable) // round 1: one refused, one unreachable + .success(contactPoint, channel3) // round 2: the set must have survived round 1 + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + // Round 2 reaches the removed host id again: the contact point's DNS still lists it. + NodeInfo round2Info = + DefaultNodeInfo.builder().withEndPoint(channel3.getEndPoint()).withHostId(hostId).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel3)) + .thenReturn(CompletableFuture.completedFuture(round2Info)); + // So that a round which wrongly stopped refusing succeeds rather than NPEs, and the assertions + // below fail on the behaviour rather than on the stub. + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); + + mockQueryPlan(contactPoint); + channel1.close(); + + factoryHelper.waitForCalls(contactPoint, 2); + verify(channel3, VERIFY_TIMEOUT).forceClose(); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(reRegistered)); + assertThat(registeredNodes).doesNotContainKey(hostId); + } + + @Test + public void should_stop_refusing_a_removed_node_after_enough_rounds_of_pure_refusals() { + // Enforcing the refusal is one thing; enforcing it for the life of the session on evidence that + // can never be rechecked is another. Every other exit from the removal set needs something this + // situation does not have -- a NodeStateEvent comes from a metadata refresh, which needs the + // control connection being restored here, and the LRU cap only evicts after 256 *further* + // removals. So if the refused host id is the only address the plan has, an unbounded version of + // the rule above would never let the session recover. After MAX_ALL_EXCLUDED_ROUNDS rounds that + // reached a live server and threw it away, the driver stops trusting the removal and lets the + // next round find out for itself. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + // One per round, for the reason given in the test above -- a reused channel arrives already + // closed and is recorded as unreachable, which would clear the set early and by accident. + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + DriverChannel channel4 = newMockDriverChannel(4); + DriverChannel channel5 = newMockDriverChannel(5); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .success(contactPoint, channel2) // round 1: refused, budget 1 of 3 + .success(contactPoint, channel3) // round 2: refused, budget 2 of 3 + .success(contactPoint, channel4) // round 3: refused, budget spent -- set cleared + .success(contactPoint, channel5) // round 4: nothing left refusing it + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + // node2 is removed and never reports a state again, so nothing can un-remove it the usual way. + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + for (DriverChannel channel : new DriverChannel[] {channel2, channel3, channel4, channel5}) { + NodeInfo info = + DefaultNodeInfo.builder().withEndPoint(channel.getEndPoint()).withHostId(hostId).build(); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(info)); + } + // As the real MetadataManager does for a host id absent from metadata: mint a fresh node and + // publish it, so the accepted round does not immediately drop the node again for being absent. + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); + + mockQueryPlan(contactPoint); + channel1.close(); + + // Then -- the fourth round is let through, rather than the session being stuck refusing the one + // address it can still reach. + factoryHelper.waitForCalls(contactPoint, 4); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(reRegistered)); + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel5)); + // And the rounds before it really were refused, rather than the set having been dropped early. + verify(channel2, VERIFY_TIMEOUT).forceClose(); + verify(channel3, VERIFY_TIMEOUT).forceClose(); + verify(channel4, VERIFY_TIMEOUT).forceClose(); + } + + @Test + public void should_not_spend_the_refusal_budget_on_rounds_that_tried_nothing() { + // The budget above is spent by a round that reached a live server and threw it away: repeating + // that learns nothing, so after enough of them the driver stops trusting the removal. A round + // whose plan was empty to begin with is not that round. It reached nothing and refused nothing, + // which is exactly why #anyNodeUnreachable declines to clear on it -- it has no standing either + // way -- and counting it against the budget would discard the set on the one kind of evidence + // both branches agree confers none. + // + // Empty plans are reachable, and they arrive here looking like every other failure: reconnect() + // hands connect() a null error list, the empty queue polls null immediately, and + // AllNodesFailedException.fromErrors(null) answers a NoNodeAvailableException whose error map + // is empty. Both predicates walk that map, so both see nothing. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .success(contactPoint, channel2) // the single round that has anything to try + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + // node2 is removed and never reports a state again, so nothing can un-remove it the usual way. + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + NodeInfo refusedInfo = + DefaultNodeInfo.builder().withEndPoint(channel2.getEndPoint()).withHostId(hostId).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(refusedInfo)); + // As the real MetadataManager would, were the node ever allowed through -- so that a failure + // here shows up as the node being accepted, not as a stub falling over. + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); + + // Comfortably more empty rounds than the budget, and the contact point offered on exactly one + // round -- every round after it is empty again, so nothing races the assertions below. + int emptyRoundsBefore = 5; + AtomicInteger round = new AtomicInteger(); + when(loadBalancingPolicyWrapper.newControlReconnectionQueryPlan()) + .thenAnswer( + i -> { + ConcurrentLinkedQueue plan = new ConcurrentLinkedQueue<>(); + if (round.getAndIncrement() == emptyRoundsBefore) { + plan.offer(contactPoint); + } + return plan; + }); + + channel1.close(); + + // Then -- the refusal outlived all of those empty rounds. Had they spent the budget, the set + // would have been cleared long before this round and the node let through. + factoryHelper.waitForCalls(contactPoint, 1); + verify(channel2, VERIFY_TIMEOUT).forceClose(); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(reRegistered)); + assertThat(registeredNodes).doesNotContainKey(hostId); + } + + @Test + public void should_accept_a_contact_point_that_resolves_to_a_re_added_node() { + // The mirror case: a removal is not a life sentence. A host id that reports any state again is + // no longer removed, so a contact point may lead back to it -- otherwise a node that bounced + // out + // and back would be blocked for the rest of the session. + UUID hostId = node2.getHostId(); + DriverChannel channel1 = newMockDriverChannel(1); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + mockQueryPlan(contactPoint); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel1).build(); + + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + // ... and back: the driver sees it again, which clears the removal. + eventBus.fire(NodeStateEvent.added(node2)); + + NodeInfo resolvedInfo = + DefaultNodeInfo.builder().withEndPoint(channel1.getEndPoint()).withHostId(hostId).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel1)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())).thenReturn(CompletableFuture.completedFuture(node2)); + + // When + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then + assertThatStage(initFuture).isSuccess(); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(node2)); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_return_control_node_after_init() { + // Given + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + // Before init, controlNode should be null + assertThat(controlConnection.controlNode()).isNull(); + + // When + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + + // Then — after init, controlNode should be set + assertThatStage(initFuture).isSuccess(); + await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); + assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node1.getHostId()); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_clear_control_node_on_channel_close_and_restore_after_reconnect() { + // Given + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + CompletableFuture channel2Future = new CompletableFuture<>(); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) + .failure(node1, "mock failure") + .pending(node2, channel2Future) + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); + assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node1.getHostId()); + + // When — channel closes, reconnection starts but pending on node2 + channel1.close(); + verify(reconnectionSchedule, VERIFY_TIMEOUT).nextDelay(); + factoryHelper.waitForCall(node1); // fails + factoryHelper.waitForCall(node2); // pending + + // Then — controlNode should be null during reconnection window + assertThat(controlConnection.controlNode()).isNull(); + + // Complete the pending reconnection + channel2Future.complete(channel2); + + // After reconnection, controlNode should be set to node2 + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel2)); + await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); + assertThat(controlConnection.controlNode().getHostId()).isEqualTo(node2.getHostId()); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_not_reconnect_on_event_for_non_control_node() { + // Given — init with node1 (has hostId) + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + await().untilAsserted(() -> assertThat(controlConnection.controlNode()).isNotNull()); + + // When — fire a distance event for node2 (not the control node) + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, node2)); + + // Then — should NOT trigger reconnection (channel stays the same) + await() + .during(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel1)); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_reconnect_when_control_node_removed_from_metadata_after_reconnect() { + // Given — init with node1 + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .success(node2, channel2) // first reconnect + .failure(node2, "decommissioned") // second reconnect: node2 fails + .success(node1, channel3) // second reconnect: falls back to node1 + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + // Remove node2 from metadata before triggering reconnect + // (simulating node2 was decommissioned during the outage) + registeredNodes.remove(node2.getHostId()); + + // When — channel1 goes down, reconnects to node2 + mockQueryPlan(node2, node1); + channel1.close(); + + // Then — onSuccessfulReconnect detects node2 is gone → triggers second reconnection + // which eventually connects to node1 (channel3) + factoryHelper.waitForCall(node2); // first reconnect + + // onSuccessfulReconnect detects node2 is removed from metadata → force-closes channel2 + verify(channel2, VERIFY_TIMEOUT).forceClose(); + + factoryHelper.waitForCall(node2); // second reconnect: node2 fails + factoryHelper.waitForCall(node1); // second reconnect: node1 succeeds + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel3)); + // channelOpened(node1) fires twice: once during init, once during second reconnect + verify(eventBus, timeout(500).times(2)).fire(ChannelEvent.channelOpened(node1)); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_not_call_getChannelNodeInfo_for_metadata_node_with_hostId() { + // Given — node1 already has a hostId (it's a metadata node, not a contact point) + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + // When + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + + // Then — resolveChannelNodeIfNeeded should short-circuit + assertThatStage(initFuture).isSuccess(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + verify(topologyMonitor, never()).getChannelNodeInfo(any(DriverChannel.class)); + + factoryHelper.verifyNoMoreCalls(); + } + + @Test + public void should_report_auth_failure_when_every_contact_point_failed_on_authentication() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, authError(node2)))); + + assertThat(ControlConnection.isAuthFailure(error)).isTrue(); + } + + @Test + public void should_not_report_auth_failure_when_an_address_failed_on_something_else() { + // One entry no longer means one address: ChannelFactory expands a contact-point hostname to + // every address it resolves to and reports one failure for the name, with the other addresses' + // failures suppressed. Reading only the top-level throwable would call this an authentication + // failure and tell the operator their credentials are wrong, when the name's other records are + // simply unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of(new SimpleEntry<>(node1, withUnreachableSibling))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_report_auth_failure_even_when_a_node_was_excluded() { + // An excluded node was never asked for credentials, so it is evidence of nothing. Letting it + // veto the verdict would hide a genuine credential problem behind one node that happened to be + // ignored or forced down -- and every exclusion path records an entry now, so this is the + // ordinary shape of a round rather than a corner. + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>( + node2, new ControlConnection.ExcludedNodeException("node became ignored")))); + + assertThat(ControlConnection.isAuthFailure(error)).isTrue(); + } + + @Test + public void should_report_auth_failure_when_one_contact_point_went_excluded_then_auth() { + // One entry, one contact-point name, two addresses: the first belongs to a node this + // connection may not use and is refused inside the connect hook, the second reaches a server + // that rejects the credentials. ChannelFactory surfaces the authentication failure with the + // refusal attached as suppressed -- wrapped the way finishCandidate wraps a hook rejection. + // + // Skipping the entry outright would drop the only evidence the round has, so it is tested + // rather than skipped; and testing it has to set the exclusion aside on that side too, or the + // suppressed refusal answers the question instead and this one contact point vetoes the + // verdict for the whole round. + Throwable withExcludedSibling = authError(node1); + withExcludedSibling.addSuppressed( + new ConnectionInitException( + "Connect hook rejected the channel", + new CompletionException( + new ControlConnection.ExcludedNodeException("node was removed")))); + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of(new SimpleEntry<>(node1, withExcludedSibling))); + + assertThat(ControlConnection.isAuthFailure(error)).isTrue(); + } + + @Test + public void should_not_report_auth_failure_when_a_contact_point_went_refused_then_auth() { + // The same shape, but the sibling was unreachable rather than refused. That is a real + // connectivity failure and must still veto: "authentication is what is wrong here" would be + // false for half of what the name resolves to. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed( + new ConnectionInitException( + "Connect failed", new CompletionException(new ConnectException("connection refused")))); + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of(new SimpleEntry<>(node1, withUnreachableSibling))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_not_report_auth_failure_when_every_node_was_excluded() { + // Nothing was tried, so there is no verdict to report. + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>( + node1, new ControlConnection.ExcludedNodeException("node became ignored")), + new SimpleEntry<>( + node2, + new CompletionException( + new ControlConnection.ExcludedNodeException("node was removed"))))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_not_report_auth_failure_when_a_node_failed_on_something_else() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, new ConnectException("connection refused")))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_try_next_node_if_the_node_behind_a_contact_point_is_excluded() { + // Given — init on node1, then a reconnection through a contact point, the way the fallback + // plan appends the original contact points to every reconnection round. + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) + .success(contactPoint, channel2) + .success(node1, channel3) + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture) + .isSuccess(v -> assertThat(controlConnection.channel()).isEqualTo(channel1)); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(node1)); + + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenAnswer( + invocation -> { + registeredNodes.put(resolvedNode.getHostId(), resolvedNode); + return CompletableFuture.completedFuture(resolvedNode); + }); + + // The backend that contact point reaches is one the policy has excluded. The event names the + // metadata node, because that is the only instance events ever name: the contact point the + // plan offered is an ephemeral object that is never their subject, so the guards that run + // before the handshake have nothing to match it against. + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When + mockQueryPlan(contactPoint, node1); + channel1.close(); + + // Then — the channel is abandoned once the handshake says which node is behind it, and the + // next node is tried. Parking here instead would be permanent: an unchanged distance fires no + // further event, so nothing downstream would ever dislodge it. + verify(reconnectionSchedule, VERIFY_TIMEOUT).nextDelay(); + factoryHelper.waitForCall(contactPoint); + verify(channel2, VERIFY_TIMEOUT).forceClose(); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(resolvedNode)); + factoryHelper.waitForCall(node1); + } + + @Test + public void should_not_report_auth_failure_for_a_throwable_with_no_per_node_breakdown() { + // Only an AllNodesFailedException carries the per-node errors this question is answered from. + // Anything else has nothing to say about the credentials, and answering yes would send an + // operator whose connection was refused off to check their authentication configuration. + assertThat(ControlConnection.isAuthFailure(new ConnectException("connection refused"))) + .isFalse(); + } + + @Test + public void should_treat_a_lone_authentication_failure_as_auth_only() { + assertThat(ChannelFactory.isAuthOnly(authError(node1))).isTrue(); + } + + @Test + public void + should_not_treat_an_authentication_failure_with_other_suppressed_causes_as_auth_only() { + // What the per-node log line is decided on. ChannelFactory#surfacedFailure deliberately + // promotes + // an authentication failure over transport ones, so the top-level throwable of a hostname whose + // records failed [refused, refused, auth] is the auth error -- and reporting that as "an + // authentication error" would say nothing about two thirds of the deployment being unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + + assertThat(ChannelFactory.isAuthOnly(withUnreachableSibling)).isFalse(); + } + + @Test + public void should_not_treat_a_non_authentication_failure_as_auth_only() { + assertThat(ChannelFactory.isAuthOnly(new ConnectException("connection refused"))).isFalse(); + } + + @Test + public void should_never_pair_captured_node_info_with_a_channel_it_did_not_come_from() { + // The candidate loop can leave a stranded hook behind: an attempt abandoned on the hook timeout + // whose system.local response arrives anyway, and writes to the holder after the accepted + // candidate has. Held in two separate volatile fields, that late write lands between the + // accepted candidate's two writes, and a reader in that window takes the rejected candidate's + // node info as the accepted channel's -- registering the wrong host id and endpoint for the + // node + // the control connection is actually talking to. + // + // Published as one reference, that window does not exist: a read either sees a whole (channel, + // info) pair or the previous whole pair, so the worst a late write can do is make the read miss + // and fall back to querying the open channel again. Asserted directly rather than raced at, + // because a race cannot demonstrate the absence of an interleaving -- and a spinning writer + // against a property that holds structurally is a test that cannot fail. + ControlConnection.NodeInfoHolder holder = new ControlConnection.NodeInfoHolder(); + DriverChannel accepted = newMockDriverChannel(1); + DriverChannel abandoned = newMockDriverChannel(2); + NodeInfo acceptedInfo = mock(NodeInfo.class); + NodeInfo abandonedInfo = mock(NodeInfo.class); + + assertThat(holder.getFor(accepted)).isNull(); + + holder.set(accepted, acceptedInfo); + assertThat(holder.getFor(accepted)).isSameAs(acceptedInfo); + // Never the other channel's, even though it is the only info the holder has. + assertThat(holder.getFor(abandoned)).isNull(); + + // A stranded hook's late write replaces the pair wholesale, so the accepted channel's read now + // misses -- which is the fallback path, not a mispairing. + holder.set(abandoned, abandonedInfo); + assertThat(holder.getFor(abandoned)).isSameAs(abandonedInfo); + assertThat(holder.getFor(accepted)).isNull(); + } + + @Test + public void should_report_why_it_gave_up_when_the_node_behind_a_contact_point_is_excluded() { + // Given -- a plan with nothing but a contact point whose backend turns out to be excluded. The + // event names the metadata node, because that is the only instance events ever name. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(2); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + // Built outside the when(): calling a mock inside an unfinished stubbing confuses Mockito. + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenReturn(CompletableFuture.completedFuture(resolvedNode)); + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When -- init, with no other node to fall back to. + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then -- the exclusion is recorded as this node's error, so the operator is told why rather + // than getting a bare NoNodeAvailableException with no cause at all. + assertThatStage(initFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(AllNodesFailedException.class); + assertThat(((AllNodesFailedException) error).getAllErrors()) + .hasEntrySatisfying( + resolvedNode, + errors -> assertThat(errors.get(0)).hasMessageContaining("ignored")); + }); + } + + @Test + public void should_reconnect_when_the_node_a_pending_channel_reached_becomes_excluded() { + // Given -- a contact point that is a hostname, so the channel's endpoint (bound by + // ChannelFactory to the address it actually reached) is the only thing that says which node is + // at the other end while the identity read is still in flight. + DefaultNode contactPoint = + DefaultNode.newContactPoint( + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.invalid", 9042)), + context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- a metadata node at the address that channel is connected to is forced down, while the + // identity read has not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, NodeState.FORCED_DOWN, TestNodeFactory.newNode(2, context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. It + // cannot compare host ids yet, and comparing the two endpoints with equals() would resolve the + // contact-point hostname inline -- a blocking DNS lookup on the admin thread, answered by + // whichever address the name lists first, which need not even be this node. + factoryHelper.waitForCall(node1); + } + + @Test + public void should_reconnect_when_a_pending_proxy_channel_reached_an_excluded_node() { + // Given -- a Cloud-style deployment. The contact point and the metadata node are SniEndPoints + // over the same proxy and server name, and the channel carries the copy ChannelFactory pinned + // to + // the proxy IP it actually reached. resolve() therefore answers an unresolved proxy name on the + // node side and a concrete IP on the channel side, and an InetSocketAddress carrying an + // InetAddress never equals one that does not -- so comparing resolve() results here would say + // "not the control node" for every Cloud, client-route and unresolved-translator deployment. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.invalid", 9142); + String serverName = "1e9a4d0c-0000-0000-0000-00000000002a"; + SniEndPoint proxyEndPoint = new SniEndPoint(proxy, serverName); + DefaultNode contactPoint = DefaultNode.newContactPoint(proxyEndPoint, context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + when(channel2.getEndPoint()) + .thenReturn(proxyEndPoint.pinTo(new InetSocketAddress("127.0.0.2", 9142))); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- the metadata node behind that same proxy is forced down, while the identity read has + // not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, + NodeState.FORCED_DOWN, + new DefaultNode(new SniEndPoint(proxy, serverName), context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. The + // proxy endpoint's identity does not key on the address at all (proxy + server name), and its + // metric prefix is derived from exactly those two, so comparing prefixes answers this without + // resolving anything. + factoryHelper.waitForCall(node1); + } + + @Test + public void should_bound_the_connect_hook_beyond_the_deadline_of_the_query_it_wraps() { + // The hook wraps a system.local read, and AdminRequestHandler already times that read out on + // CONTROL_CONNECTION_TIMEOUT -- from a value DefaultTopologyMonitor snapshotted in its + // constructor, while this reads the option live. Giving the hook the same number leaves the two + // deadlines with no defined order, and which one fires decides what the operator is told: + // "Connect hook timed out" names the wrapper, the inner DriverTimeoutException names the query + // and the node. The hook is the backstop for a stage that never completes at all -- which a + // custom TopologyMonitor can produce and the built-in one cannot -- so it has to be the looser + // of the two. + Duration configured = Duration.ofSeconds(3); + when(defaultProfile.getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT)) + .thenReturn(configured); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + assertThat(connectOptions(contactPoint).connectHookTimeout).isGreaterThan(configured); + } + + @Test + public void should_spend_the_refusal_budget_when_the_exclusion_is_only_suppressed() { + // The budget above was reachable only when the refusal was the failure the round surfaced. It + // usually is not. A contact point reports one failure for the whole name with the other + // addresses' failures attached as suppressed, and ChannelFactory#surfacedFailure ranks an + // authentication failure above a refusal -- so [excluded, bad-credentials] arrives as the + // AuthenticationException with the refusal suppressed, deterministically, every round. + // + // Both readers of the round have to look equally deep at that. #anyNodeUnreachable does, and + // correctly declines to call such a round unreachable; a shallower #anyNodeExcluded would + // decline too, and the round would fall between them -- clearing nothing, spending nothing, for + // the life of the session. Which is the deadlock the budget exists to break, reintroduced. + UUID hostId = node2.getHostId(); + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel5 = newMockDriverChannel(5); + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) // init + .failure(contactPoint, excludedThenAuth(contactPoint)) // round 1: budget 1 of 3 + .failure(contactPoint, excludedThenAuth(contactPoint)) // round 2: budget 2 of 3 + .failure(contactPoint, excludedThenAuth(contactPoint)) // round 3: spent -- set cleared + .success(contactPoint, channel5) // round 4: nothing left refusing it + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture).isSuccess(); + + // node2 is removed and never reports a state again, so nothing can un-remove it the usual way. + eventBus.fire(NodeStateEvent.removed(node2)); + registeredNodes.remove(hostId); + + NodeInfo acceptedInfo = + DefaultNodeInfo.builder().withEndPoint(channel5.getEndPoint()).withHostId(hostId).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel5)) + .thenReturn(CompletableFuture.completedFuture(acceptedInfo)); + DefaultNode reRegistered = TestNodeFactory.newNode(2, hostId, context); + when(metadataManager.registerNode(any())) + .thenAnswer( + i -> { + registeredNodes.put(hostId, reRegistered); + return CompletableFuture.completedFuture(reRegistered); + }); + + mockQueryPlan(contactPoint); + channel1.close(); + + // Then -- the fourth round is let through, so the three rounds before it really did count. + factoryHelper.waitForCalls(contactPoint, 4); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(reRegistered)); + await().untilAsserted(() -> assertThat(controlConnection.channel()).isEqualTo(channel5)); + } + + @Test + public void should_warn_about_credentials_when_something_else_is_what_surfaced() { + // advanced.connection.warn-on-init-error is false here, which is the setting that makes this + // observable: it mutes the noise of nodes that cannot be reached, and was never a switch for + // "your credentials are wrong". So a round that includes a credential rejection has to warn + // whatever else it includes -- and which of the set arrives here is surfacedFailure's choice, + // not the operator's. It ranks a node-wide failure above an authentication one, so the + // rejected login commonly travels as a suppressed exception, and a test on the type of what + // arrived would send exactly this case down the muted path. + Logger logger = (Logger) LoggerFactory.getLogger(ControlConnection.class); + Level levelBefore = logger.getLevel(); + logger.setLevel(Level.WARN); + logger.addAppender(appender); + try { + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .failure(node1, authBehindNodeWideFailure(node1)) + .success(node2, channel1) + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + factoryHelper.waitForCall(node2); + assertThatStage(initFuture).isSuccess(); + + ArgumentCaptor logs = ArgumentCaptor.forClass(ILoggingEvent.class); + verify(appender, VERIFY_TIMEOUT.atLeastOnce()).doAppend(logs.capture()); + assertThat(logs.getAllValues()) + .anySatisfy( + event -> { + assertThat(event.getLevel()).isEqualTo(Level.WARN); + assertThat(event.getFormattedMessage()).contains("authentication failed on some"); + }); + } finally { + logger.detachAppender(appender); + logger.setLevel(levelBefore); + } + } + + @Test + public void should_reconnect_when_a_pending_channel_is_pinned_to_a_hostname_contact_point() { + // Given -- a plain hostname contact point, which is now kept unresolved, and a channel carrying + // the copy ChannelFactory pinned to the one address it reached. So resolve() answers an + // unresolved name on the node side and a concrete IP on the channel side: the resolve() + // comparison cannot match, and equals() is the one thing that must not be asked, because + // DefaultEndPoint#equals resolves the unresolved side -- a blocking DNS lookup on the admin + // thread, on whichever address the name happens to list first (issue #1006). + InetSocketAddress hostname = InetSocketAddress.createUnresolved("db.example.invalid", 9042); + DefaultEndPoint hostnameEndPoint = new DefaultEndPoint(hostname); + DefaultNode contactPoint = DefaultNode.newContactPoint(hostnameEndPoint, context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + when(channel2.getEndPoint()) + .thenReturn(hostnameEndPoint.pinTo(new InetSocketAddress("127.0.0.2", 9042))); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- that same contact point is forced down while the identity read has not come back, so + // its node still holds the unresolved name. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, NodeState.FORCED_DOWN, new DefaultNode(hostnameEndPoint, context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. A pin + // is excluded from the metric identity by PinnableEndPoint's contract, so the name and the + // channel pinned to one of its addresses carry the same prefix, and comparing those answers + // this without resolving anything. + factoryHelper.waitForCall(node1); + } + + /** + * One contact-point name, two addresses: the first belongs to a node this connection may not use + * and is refused inside the connect hook, the second reaches a server that rejects the + * credentials. Wrapped the way {@code ChannelFactory#finishCandidate} wraps a hook rejection, and + * surfaced the way {@code ChannelFactory#surfacedFailure} surfaces this pair. + */ + private static Throwable excludedThenAuth(Node node) { + Throwable error = authError(node); + error.addSuppressed( + new ConnectionInitException( + "Connect hook rejected the channel", + new CompletionException( + new ControlConnection.ExcludedNodeException("node was removed")))); + return error; + } + + /** + * The same shape with the ranking that hides it: the surfaced failure is node-wide, which {@code + * ChannelFactory#surfacedFailure} promotes over an authentication one, so the rejected login is + * reachable only through {@link Throwable#getSuppressed()}. + */ + private static Throwable authBehindNodeWideFailure(Node node) { + Throwable error = + new ConnectionInitException( + "Server does not support the CLIENT_ROUTES_CHANGE event type", null); + error.addSuppressed(authError(node)); + return error; + } + + private static Throwable authError(Node node) { + return new AuthenticationException(node.getEndPoint(), "mock authentication failure"); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java index b5e843d77d2..9c36cbebfee 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -29,9 +30,12 @@ import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metadata.NodeState; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.InetSocketAddress; import java.util.UUID; import org.junit.Test; import org.junit.runner.RunWith; @@ -210,6 +214,38 @@ public void should_warn_if_configured_dc_matches_no_node() { .isTrue(); } + @Test + public void should_not_warn_about_dc_mismatch_when_the_only_real_node_matches_configured_dc() { + // Given — CUSTOMER-588. A contact point given as a hostname is represented, before the control + // connection resolves it, by an ephemeral placeholder Node (built by + // MetadataManager#addContactPoints via DefaultNode#newContactPoint) whose datacenter is always + // null: it is never populated, because real topology is attached to a *different* Node object + // matched by hostId (see MetadataManager#registerNode). + // + // The removed OptionalLocalDcHelper#checkLocalDatacenterCompatibility compared the configured + // local DC against *those* placeholders, so it warned unconditionally whenever a local DC was + // configured, no matter where the contact points actually were. Here the only node carrying + // real, resolved metadata (node1) genuinely is in the configured local DC ("dc1", per base + // setup). + DefaultNode ephemeralContactPointNode = + DefaultNode.newContactPoint( + new DefaultEndPoint(new InetSocketAddress("127.0.0.9", 9042)), context); + when(metadataManager.getContactPoints()).thenReturn(ImmutableSet.of(ephemeralContactPointNode)); + DefaultLoadBalancingPolicy policy = createPolicy(); + + // When + policy.init(ImmutableMap.of(UUID.randomUUID(), node1), distanceReporter); + + // Then — no WARN at all. The retained check inspects the resolved node map, where node1 + // matches. + // Asserting that nothing is warned, rather than that one particular message is absent, also + // catches a regression that brings the false positive back under different wording. + // should_warn_if_configured_dc_matches_no_node is the positive control for this same appender, + // so a silent capture failure cannot make this pass by accident. + verify(appender, never()).doAppend(argThat(event -> event.getLevel() == Level.WARN)); + assertThat(policy.getLocalDatacenter()).isEqualTo("dc1"); + } + @NonNull protected DefaultLoadBalancingPolicy createPolicy() { return new DefaultLoadBalancingPolicy(context, DriverExecutionProfile.DEFAULT_NAME); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefreshTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefreshTest.java index 8d337bcc7e3..2291cf68ba8 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefreshTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/AddNodeRefreshTest.java @@ -26,6 +26,7 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Collections; import java.util.Map; @@ -92,10 +93,14 @@ public void should_not_add_existing_node_with_same_id_and_endpoint() { DefaultMetadata oldMetadata = new DefaultMetadata( ImmutableMap.of(node1.getHostId(), node1), Collections.emptyMap(), null, null); + // Carrying the broadcast RPC address it would really arrive with: that is what the guard + // compares, and findInPeers never builds a NodeInfo without one. DefaultNodeInfo newNodeInfo = DefaultNodeInfo.builder() .withHostId(node1.getHostId()) .withEndPoint(node1.getEndPoint()) + .withBroadcastRpcAddress( + node1.getBroadcastRpcAddress().orElseThrow(AssertionError::new)) .withDatacenter("dc1") .withRack("rack2") .build(); @@ -144,4 +149,113 @@ public void should_add_existing_node_with_same_id_but_different_endpoint() { assertThat(node1.getSchemaVersion()).isEqualTo(newSchemaVersion); assertThat(result.events).containsExactly(TopologyEvent.suggestUp(newBroadcastRpcAddress)); } + + @Test + public void should_not_add_existing_node_whose_endpoint_only_changed_representation() { + // The same node described two ways by two of the driver's own code paths. + // DefaultTopologyMonitor#connectedNodeEndPoint gives the control node an identity built from + // the address its channel reached, while that node's peers row would give it whatever the + // configured AddressTranslator returns -- an unresolved name, under a translator with + // resolve-addresses = false. Cassandra reports a restart as a NEW_NODE even when the host id + // did not change, so those two forms are compared here, and nothing about the node's addressing + // has moved. + // + // Comparing endpoints reports a change on every such event: EndPoint#equals would resolve the + // unresolved side (a blocking lookup on the admin event loop, answered by whichever address the + // resolver lists first -- issue #1006) and PinnableEndPoint#sameIdentity compares metric + // identity, which is precisely what differs. Either one copies the peers-derived endpoint in, + // flipping the node's metric prefix and clearing its series, and asks for a pool bounce. + // + // Derived from one InetAddress and spelled as its own literal, so that nothing here resolves: + // rebuilding a literal parses. + InetAddress loopback = InetAddress.getLoopbackAddress(); + InetSocketAddress broadcastRpcAddress = new InetSocketAddress(loopback, 9042); + DefaultNode existing = new DefaultNode(new DefaultEndPoint(broadcastRpcAddress), context); + UUID hostId = Uuids.random(); + existing.hostId = hostId; + existing.broadcastRpcAddress = broadcastRpcAddress; + DefaultMetadata oldMetadata = + new DefaultMetadata(ImmutableMap.of(hostId, existing), Collections.emptyMap(), null, null); + + DefaultEndPoint asName = + new DefaultEndPoint(InetSocketAddress.createUnresolved(loopback.getHostAddress(), 9042)); + DefaultNodeInfo newNodeInfo = + DefaultNodeInfo.builder() + .withHostId(hostId) + .withEndPoint(asName) + .withBroadcastRpcAddress(broadcastRpcAddress) + .build(); + + MetadataRefresh.Result result = + new AddNodeRefresh(newNodeInfo).compute(oldMetadata, false, context); + + assertThat(result.events).isEmpty(); + assertThat(existing.getEndPoint()).isEqualTo(new DefaultEndPoint(broadcastRpcAddress)); + } + + @Test + public void should_ignore_new_node_info_without_a_broadcast_rpc_address() { + // TopologyMonitor#getNewNodeInfo is an extension point, and nothing stops an implementation + // returning a NodeInfo with no broadcast RPC address -- findInPeers always supplies one, but a + // custom monitor need not. An absent address satisfies the inequality against the existing + // node's present one, so before the guard this reached Optional#get() and threw + // NoSuchElementException out of MetadataRefresh#compute, which MetadataManager#apply does not + // catch. An `assert` cannot hold that line: it is not enabled in production. + InetAddress loopback = InetAddress.getLoopbackAddress(); + InetSocketAddress broadcastRpcAddress = new InetSocketAddress(loopback, 9042); + DefaultNode existing = new DefaultNode(new DefaultEndPoint(broadcastRpcAddress), context); + UUID hostId = Uuids.random(); + existing.hostId = hostId; + existing.broadcastRpcAddress = broadcastRpcAddress; + DefaultMetadata oldMetadata = + new DefaultMetadata(ImmutableMap.of(hostId, existing), Collections.emptyMap(), null, null); + + // No withBroadcastRpcAddress(...) call, and a different endpoint, so nothing else could make + // this a no-op. + DefaultNodeInfo newNodeInfo = + DefaultNodeInfo.builder() + .withHostId(hostId) + .withEndPoint(new DefaultEndPoint(new InetSocketAddress(loopback, 9043))) + .build(); + + MetadataRefresh.Result result = + new AddNodeRefresh(newNodeInfo).compute(oldMetadata, false, context); + + assertThat(result.events).isEmpty(); + // Not adopted either: there is nothing to compare, so the refresh changes nothing at all. + assertThat(existing.getEndPoint()).isEqualTo(new DefaultEndPoint(broadcastRpcAddress)); + } + + @Test + public void should_add_existing_node_that_moved_behind_a_shared_translator_name() { + // The mirror, and the direction the endpoint comparison lost outright. A translator that hands + // back a name maps every node it covers to the same endpoint -- FixedHostNameAddressTranslator + // returns one advertised hostname for the whole cluster -- so a node that really did change its + // broadcast RPC address arrives with an endpoint identical to the one it already had. Comparing + // endpoints calls that "unchanged" and the node's pool is never told to reconnect, on the one + // event that exists to tell it. + DefaultEndPoint advertised = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + DefaultNode existing = new DefaultNode(advertised, context); + UUID hostId = Uuids.random(); + existing.hostId = hostId; + existing.broadcastRpcAddress = new InetSocketAddress("127.0.0.1", 9042); + DefaultMetadata oldMetadata = + new DefaultMetadata(ImmutableMap.of(hostId, existing), Collections.emptyMap(), null, null); + + InetSocketAddress movedTo = new InetSocketAddress("127.0.0.2", 9042); + DefaultNodeInfo newNodeInfo = + DefaultNodeInfo.builder() + .withHostId(hostId) + .withEndPoint(advertised) + .withBroadcastRpcAddress(movedTo) + .withRack("rack2") + .build(); + + MetadataRefresh.Result result = + new AddNodeRefresh(newNodeInfo).compute(oldMetadata, false, context); + + assertThat(result.events).containsExactly(TopologyEvent.suggestUp(movedTo)); + assertThat(existing.getRack()).isEqualTo("rack2"); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index f31dd2861ed..298ebf6c082 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -18,18 +18,22 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import java.io.UncheckedIOException; +import io.netty.channel.local.LocalAddress; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.net.UnknownHostException; import java.util.UUID; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @@ -66,20 +70,23 @@ public void should_fallback_when_resolve_returns_null() throws UnknownHostExcept } @Test - public void should_wrap_io_exceptions_in_unchecked_io_exception() throws UnknownHostException { + public void should_return_the_route_address_unresolved() { + // The route hostname is handed over unresolved on purpose: ChannelFactory resolves it through + // Netty's AddressResolverGroup, so a custom resolver applies to client routes too and no DNS + // lookup runs on the admin event loop that connect() is called from. UUID hostId = UUID.randomUUID(); - when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); ClientRoutesEndPoint ep = new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - assertThatThrownBy(ep::resolve) - .isInstanceOf(UncheckedIOException.class) - .hasCauseInstanceOf(UnknownHostException.class); + assertThat(ep.resolve()).isSameAs(route); + assertThat(((InetSocketAddress) ep.resolve()).isUnresolved()).isTrue(); } @Test - public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownHostException { + public void should_reflect_route_changes_on_subsequent_resolve() { UUID hostId = UUID.randomUUID(); InetSocketAddress addr1 = new InetSocketAddress("127.0.0.1", 9042); InetSocketAddress addr2 = new InetSocketAddress("10.0.0.1", 9043); @@ -96,6 +103,67 @@ public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownH assertThat(ep.resolve()).isEqualTo(addr2); } + // ---- pinTo() ------------------------------------------------------------ + + @Test + public void pin_to_should_stop_consulting_the_topology_monitor() { + UUID hostId = UUID.randomUUID(); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // No lookup at all -- that is the point: DefaultTopologyMonitor#savePort and the SSL factories + // read the channel's endpoint, and must not trigger a blocking re-resolution there. + verify(topologyMonitor, never()).resolve(hostId); + // Identity is keyed off the host id, so the pinned copy is still the same node. + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((ClientRoutesEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolve() hands out the route's hostname unresolved, and ChannelFactory passes an address + // straight back when the user disabled the resolver or a custom one declines it -- so that + // hostname can come back here. Pinning it would freeze the endpoint on a name that still + // re-expands on every connect, and silence the route lookup for good, since resolve() + // short-circuits once pinned. Both siblings return `this` for the same input. + UUID hostId = UUID.randomUUID(); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThat(endPoint.pinTo(route)).isSameAs(endPoint); + // Still asking the monitor, which is what would have been lost. + assertThat(endPoint.resolve()).isEqualTo(route); + verify(topologyMonitor, atLeastOnce()).resolve(hostId); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // Mirror DefaultEndPoint: an address that cannot be held in an InetSocketAddress field (e.g. + // the local transport used by unit tests) skips pinning rather than failing the connection. + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + // ---- equals / hashCode -------------------------------------------------- @Test @@ -163,6 +231,130 @@ public void should_format_ipv6_metric_prefix() throws Exception { assertThat(ep.asMetricPrefix()).isEqualTo("0:0:0:0:0:0:0:1_" + hostId); } + // ---- addressesAreInterchangeable() -------------------------------------- + + @Test + public void should_report_a_routes_addresses_as_interchangeable() { + // The proxy routes by server name, so every address the route's hostname maps to reaches this + // same node and connections may be spread across them. + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)) + .thenReturn(InetSocketAddress.createUnresolved("route.example.com", 9042)); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThat(ep.addressesAreInterchangeable(ep.resolve())).isTrue(); + } + + @Test + public void should_defer_to_the_fallback_when_there_is_no_route() { + // Without a route, resolve() is the fallback endpoint's answer, so whether *those* addresses + // are + // interchangeable is not this class's to claim. The usual fallback is a DefaultEndPoint built + // from a translated broadcast address, which says no -- and an AddressTranslator that returns a + // name (SubnetAddressTranslator does by default) is exactly the case where spreading would land + // one node's channels on different hosts while routing and metrics attribute them to one node. + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)).thenReturn(null); + EndPoint staticFallback = + new DefaultEndPoint(InetSocketAddress.createUnresolved("peer.example.com", 9042)); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, staticFallback); + + assertThat(ep.addressesAreInterchangeable(ep.resolve())).isFalse(); + } + + @Test + public void should_defer_to_a_fallback_that_is_itself_interchangeable() { + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)).thenReturn(null); + EndPoint sniFallback = + new SniEndPoint(InetSocketAddress.createUnresolved("proxy.example.com", 9042), "server"); + + ClientRoutesEndPoint ep = new ClientRoutesEndPoint(topologyMonitor, hostId, null, sniFallback); + + assertThat(ep.addressesAreInterchangeable(ep.resolve())).isTrue(); + } + + @Test + public void should_not_throw_from_addresses_are_interchangeable_when_the_monitor_is_closed() { + // Same reasoning as resolve(): a closed monitor means "no route", not a failed connect. This is + // asked on the connect path, so throwing here would fail the attempt outright. It cannot throw + // any more for a structural reason rather than a caught one -- the answer comes off the address + // resolve() already returned, so the monitor is not consulted a second time at all. + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)) + .thenThrow(new IllegalStateException("Topology monitor is closed")); + EndPoint staticFallback = + new DefaultEndPoint(InetSocketAddress.createUnresolved("peer.example.com", 9042)); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, staticFallback); + + // resolve() catches the closure and answers with the fallback; that address is what the connect + // would dial, so that is what this is asked about. + SocketAddress chosen = ep.resolve(); + assertThat(chosen).isEqualTo(staticFallback.resolve()); + Mockito.clearInvocations(topologyMonitor); + + assertThat(ep.addressesAreInterchangeable(chosen)).isFalse(); + verify(topologyMonitor, never()).resolve(Mockito.any()); + } + + @Test + public void should_not_spread_a_fallback_address_when_a_route_appears_mid_connect() { + // The race this signature exists to close. ChannelFactory.connect() calls resolve() and then, + // separately, asks whether the addresses may be spread across; the route cache is swapped from + // the routes-query thread, so a CLIENT_ROUTES_CHANGE can land between the two. In the direction + // that matters the address already chosen is the *fallback's* -- under a translator returning a + // name, one that may denote several hosts -- and a route appearing afterwards would authorise + // shuffling it, landing one node's channels on different servers. Deciding from the address + // that was actually handed out makes the second read irrelevant. + UUID hostId = UUID.randomUUID(); + EndPoint translatedName = + new DefaultEndPoint(InetSocketAddress.createUnresolved("rack1.example.com", 9042)); + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, translatedName); + + // Read #1: no route, so the connect will dial the fallback's name. + when(topologyMonitor.resolve(hostId)).thenReturn(null); + SocketAddress chosen = ep.resolve(); + assertThat(chosen).isEqualTo(translatedName.resolve()); + + // A route is published in the window between the two calls. + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); + // The cache really has changed -- a fresh resolve() would now hand out the route. + assertThat(ep.resolve()).isEqualTo(route); + + // But the address this connect settled on is still the fallback's, and that is what decides. + assertThat(ep.addressesAreInterchangeable(chosen)).isFalse(); + } + + @Test + public void should_answer_without_consulting_the_monitor_once_pinned() { + // resolve() short-circuits on the pinned address and stops consulting the route cache, so this + // must not consult it either -- a pinned endpoint denotes the one server it is fixed to. + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)) + .thenReturn(InetSocketAddress.createUnresolved("route.example.com", 9042)); + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + // Unpinned it does consult the cache, and says yes -- so the assertion below is about pinning, + // not about the route having gone away. + assertThat(ep.addressesAreInterchangeable(ep.resolve())).isTrue(); + + ClientRoutesEndPoint pinnedEp = + (ClientRoutesEndPoint) ep.pinTo(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(pinnedEp).isNotSameAs(ep); + Mockito.reset(topologyMonitor); + + assertThat(pinnedEp.addressesAreInterchangeable(pinnedEp.resolve())).isFalse(); + verify(topologyMonitor, never()).resolve(Mockito.any()); + } + // ---- toString() --------------------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index a1ba4617ef5..43f9ad0afe9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -28,6 +28,8 @@ import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; import com.datastax.oss.driver.internal.core.channel.DriverChannel; @@ -66,6 +68,8 @@ public class ClientRoutesTopologyMonitorTest { @Mock private ControlConnection controlConnection; @Mock private DriverConfig driverConfig; @Mock private DriverExecutionProfile defaultProfile; + @Mock private MetadataManager metadataManager; + @Mock private Metadata metadata; private TestableClientRoutesTopologyMonitor handler; @@ -113,6 +117,11 @@ String lastCapturedQuery() { return capturedQueries.get(capturedQueries.size() - 1); } + EndPoint buildNodeEndPointForTest( + AdminRow row, InetSocketAddress broadcastRpcAddress, EndPoint localEndPoint) { + return buildNodeEndPoint(row, broadcastRpcAddress, localEndPoint); + } + @Override @NonNull protected CompletionStage runAdminQuery( @@ -170,6 +179,33 @@ public void should_return_null_for_unknown_host_id() throws UnknownHostException assertThat(handler.resolve(UUID.randomUUID())).isNull(); } + // ---- buildNodeEndPoint() --------------------------------------------- + + @Test + public void should_not_nest_a_client_routes_endpoint_as_its_own_fallback() { + // For the system.local row the superclass hands back the control channel's own endpoint, which + // in a client-routes deployment is already a ClientRoutesEndPoint -- and a *pinned* one, since + // ChannelFactory binds the channel's endpoint to the address it reached. Nesting it would make + // this endpoint's route-less fallback a frozen proxy IP instead of the static address the chain + // is supposed to bottom out at, and would add a level per control reconnect, each retaining a + // topology monitor and an O(depth) walk in resolve(). + UUID hostId = UUID.randomUUID(); + EndPoint staticFallback = new DefaultEndPoint(new InetSocketAddress("10.0.0.1", 9042)); + EndPoint channelEndPoint = + new ClientRoutesEndPoint(handler, hostId, null, staticFallback) + .pinTo(new InetSocketAddress("192.168.0.1", 9042)); + AdminRow localRow = Mockito.mock(AdminRow.class); + when(localRow.contains("peer")).thenReturn(false); + when(localRow.getUuid("host_id")).thenReturn(hostId); + + EndPoint built = handler.buildNodeEndPointForTest(localRow, null, channelEndPoint); + + assertThat(built).isInstanceOf(ClientRoutesEndPoint.class); + assertThat(((ClientRoutesEndPoint) built).getFallbackEndPoint()) + .isNotInstanceOf(ClientRoutesEndPoint.class) + .isSameAs(staticFallback); + } + @Test public void should_resolve_known_host_id() throws UnknownHostException { UUID hostId = UUID.randomUUID(); @@ -194,14 +230,21 @@ public void should_throw_after_close() { } @Test - public void should_throw_for_unresolvable_hostname() { + public void should_not_look_up_the_route_hostname() { UUID hostId = UUID.randomUUID(); - // Use a hostname guaranteed not to resolve + // A hostname guaranteed not to resolve: this must still succeed, because resolve() is a pure + // in-memory cache lookup that hands the name over unresolved. ChannelFactory resolves it later + // through Netty's AddressResolverGroup, so a custom resolver applies to client routes too and + // nothing blocks the admin event loop here. handler.setRoutes( ImmutableMap.of( hostId, new ClientRouteRecord(hostId, "this.host.does.not.exist.invalid", 9042))); - assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(UnknownHostException.class); + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result.isUnresolved()).isTrue(); + assertThat(result.getHostString()).isEqualTo("this.host.does.not.exist.invalid"); + assertThat(result.getPort()).isEqualTo(9042); } @Test @@ -220,6 +263,85 @@ public void should_refresh_updates_routes() throws UnknownHostException { assertThat(handler.resolve(hostId2)).isNotNull(); } + // ---- reresolvesNodeAddresses() ------------------------------------------- + + @Test + public void should_reresolve_when_all_known_nodes_have_client_routes() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + handler.setRoutes( + ImmutableMap.of( + hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042), + hostId2, new ClientRouteRecord(hostId2, "127.0.0.2", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + + @Test + public void should_not_reresolve_when_a_known_node_has_no_client_route() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + // Only node1 has a live client route; node2 would fall back to a static endpoint. + handler.setRoutes(ImmutableMap.of(hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + + @Test + public void should_not_reresolve_when_no_nodes_are_known_yet() { + // "Every known node has a route" is vacuously true of an empty set, and answering yes there + // suppresses the contact-point reconnection fallback at the one moment it is the only way back: + // before the first node refresh, or after this monitor has removed every node. There is nothing + // for the monitor to keep fresh, so it must not claim it is keeping anything fresh. + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(Collections.emptyMap()); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + + @Test + public void should_not_reresolve_once_closed() throws Exception { + // resolve() throws IllegalStateException once the monitor is closed, and + // ClientRoutesEndPoint#resolve() catches that and quietly returns the static fallback -- so a + // closed monitor re-resolves nothing, however complete its route cache still looks. Answering + // from the cache alone would keep saying "yes" and suppress the contact-point fallback for a + // reconnection racing session shutdown. + UUID hostId = UUID.randomUUID(); + Node node = Mockito.mock(Node.class); + when(node.getHostId()).thenReturn(hostId); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId, node)); + handler.setRoutes(ImmutableMap.of(hostId, new ClientRouteRecord(hostId, "127.0.0.1", 9042))); + + // Every known node has a route, so this is the case that would otherwise report true. + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + + handler.closeAsync().toCompletableFuture().get(5, TimeUnit.SECONDS); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + // ---- Merge behavior tests ----------------------------------------------- @Test @@ -966,6 +1088,88 @@ public void should_skip_route_when_port_is_null() throws Exception { assertThat(handler.getRoutes()).doesNotContainKey(hostId); } + @Test + public void should_skip_a_zero_port_row_without_losing_the_rest_of_the_refresh() + throws Exception { + // Port 0 passes the isNull check and is legal for InetSocketAddress.createUnresolved, so a + // bare `< 0` range check lets it through -- and then ClientRouteRecord's own constructor + // throws IllegalArgumentException("port must be between 1 and 65535"). That throw happens + // inside the row loop, so it takes down the *whole* refresh: every other row in the batch is + // discarded and the cache is left as it was, for as long as the bad row stays in the table. + // Skipping the one row here, with the diagnostic that names it, is exactly what this guard + // exists for -- so the assertion that matters is that the good row survives. + UUID badHostId = UUID.randomUUID(); + UUID goodHostId = UUID.randomUUID(); + + AdminRow badRow = Mockito.mock(AdminRow.class); + when(badRow.isNull("host_id")).thenReturn(false); + when(badRow.isNull("address")).thenReturn(false); + when(badRow.isNull("port")).thenReturn(false); + when(badRow.getUuid("host_id")).thenReturn(badHostId); + when(badRow.getString("address")).thenReturn("127.0.0.1"); + when(badRow.getInteger("port")).thenReturn(0); + + AdminRow goodRow = Mockito.mock(AdminRow.class); + when(goodRow.isNull("host_id")).thenReturn(false); + when(goodRow.isNull("address")).thenReturn(false); + when(goodRow.isNull("port")).thenReturn(false); + when(goodRow.getUuid("host_id")).thenReturn(goodHostId); + when(goodRow.getString("address")).thenReturn("127.0.0.2"); + when(goodRow.getInteger("port")).thenReturn(9042); + + handler.setNextQueryResult(AdminResultTestHelper.mockResult(badRow, goodRow)); + when(controlConnection.channel()).thenReturn(Mockito.mock(DriverChannel.class)); + + handler.refresh().toCompletableFuture().get(5, TimeUnit.SECONDS); + + assertThat(handler.getRoutes()).doesNotContainKey(badHostId); + assertThat(handler.getRoutes()).containsKey(goodHostId); + assertThat(handler.getRoutes().get(goodHostId).getPort()).isEqualTo(9042); + } + + @Test + public void should_skip_an_empty_address_row_without_losing_the_rest_of_the_refresh() + throws Exception { + // The sibling of the zero-port case above, on the field the range check does not cover. + // isNull("address") is false for an empty string, so the incomplete-row guard passes it + // through, and ClientRouteRecord's constructor then throws + // IllegalArgumentException("hostname must not be empty") from inside the row loop -- taking + // the whole refresh with it, not just this row. + UUID badHostId = UUID.randomUUID(); + UUID goodHostId = UUID.randomUUID(); + + // The port stubs are lenient, not absent, and that distinction is what makes this test bind. + // The address guard runs before the port is read, so under the fix these go unused and strict + // stubbing rejects them -- but dropping them makes an unstubbed getInteger("port") answer null, + // which the "required port column is not set" guard above catches. The row would then be + // skipped for the wrong reason in *both* versions and the test would pass against the pristine + // file, proving nothing. Stubbed and lenient, the pristine code reaches the constructor. + AdminRow badRow = Mockito.mock(AdminRow.class); + when(badRow.isNull("host_id")).thenReturn(false); + when(badRow.isNull("address")).thenReturn(false); + when(badRow.getUuid("host_id")).thenReturn(badHostId); + when(badRow.getString("address")).thenReturn(""); + Mockito.lenient().when(badRow.isNull("port")).thenReturn(false); + Mockito.lenient().when(badRow.getInteger("port")).thenReturn(9042); + + AdminRow goodRow = Mockito.mock(AdminRow.class); + when(goodRow.isNull("host_id")).thenReturn(false); + when(goodRow.isNull("address")).thenReturn(false); + when(goodRow.isNull("port")).thenReturn(false); + when(goodRow.getUuid("host_id")).thenReturn(goodHostId); + when(goodRow.getString("address")).thenReturn("127.0.0.2"); + when(goodRow.getInteger("port")).thenReturn(9042); + + handler.setNextQueryResult(AdminResultTestHelper.mockResult(badRow, goodRow)); + when(controlConnection.channel()).thenReturn(Mockito.mock(DriverChannel.class)); + + handler.refresh().toCompletableFuture().get(5, TimeUnit.SECONDS); + + assertThat(handler.getRoutes()).doesNotContainKey(badHostId); + assertThat(handler.getRoutes()).containsKey(goodHostId); + assertThat(handler.getRoutes().get(goodHostId).getPort()).isEqualTo(9042); + } + // ---- Empty-result cache guard tests ------------------------------------ @Test @@ -1056,6 +1260,28 @@ public void should_build_default_endpoint_when_host_id_is_null() { assertThat(result).isSameAs(localEndPoint); } + @Test + public void should_not_give_an_unidentified_row_the_control_node_identity() { + // The same null host_id, but on the system.local row of a deployment whose control channel has + // already been pinned. The superclass hands back that channel's endpoint, which is a *pinned* + // ClientRoutesEndPoint: its equals/hashCode key off the host id alone, its metric prefix and + // toString name that node, and its resolve() is frozen on the proxy IP that connection + // reached. Returning it for a row the driver could not identify would hand back another node's + // identity, and the route cache would never be consulted for it again. + UUID controlNodeId = UUID.randomUUID(); + EndPoint staticFallback = new DefaultEndPoint(new InetSocketAddress("10.0.0.1", 9042)); + EndPoint channelEndPoint = + new ClientRoutesEndPoint(handler, controlNodeId, null, staticFallback) + .pinTo(new InetSocketAddress("192.168.0.1", 9042)); + AdminRow localRow = Mockito.mock(AdminRow.class); + when(localRow.contains("peer")).thenReturn(false); + when(localRow.getUuid("host_id")).thenReturn(null); + + EndPoint built = handler.buildNodeEndPointForTest(localRow, null, channelEndPoint); + + assertThat(built).isNotInstanceOf(ClientRoutesEndPoint.class).isSameAs(staticFallback); + } + @Test public void should_build_client_routes_endpoint_when_host_id_non_null() { // Even with empty routes cache, a ClientRoutesEndPoint is created so it can diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitorTest.java new file mode 100644 index 00000000000..fa0f34d7c69 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitorTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfig; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.control.ControlConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.UUID; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class CloudTopologyMonitorTest { + + @Mock private InternalDriverContext context; + @Mock private ControlConnection controlConnection; + @Mock private DriverConfig config; + @Mock private DriverExecutionProfile defaultProfile; + + @Before + public void setup() { + when(context.getSessionName()).thenReturn("test"); + when(context.getControlConnection()).thenReturn(controlConnection); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(defaultProfile); + when(defaultProfile.getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT)) + .thenReturn(Duration.ofSeconds(5)); + when(defaultProfile.getBoolean(DefaultDriverOption.RECONNECT_ON_INIT)).thenReturn(false); + } + + @Test + public void should_keep_endpoint_identity_stable_when_the_proxy_address_caches_a_name() + throws Exception { + // The drift itself, observed end to end -- the companion to the test above, which pins the + // normalization without depending on the environment. + // + // SniEndPoint promises a stable equals/hashCode/asMetricPrefix and derives all three from the + // proxy address's host string. When the InetSocketAddress was built from an InetAddress that + // carries no hostName -- getByAddress(bytes), as below -- getHostString() renders that + // address's lazily-populated field, and anything calling getHostName() fills it in. + // DefaultSslEngineFactory does exactly that under the default allow-dns-reverse-lookup-san. + // (new InetSocketAddress("host", port) is *not* affected: getByName populates hostName + // eagerly. That distinction is why this test constructs the address the way it does.) + // + // This monitor rebuilds every node's endpoint on every topology refresh, so re-deriving the + // string each time would move the whole cluster's per-node metrics at once, mid-session, and + // make endpoints built before and after compare unequal. + InetSocketAddress proxy = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 9042); + // Environment-dependent, unavoidably. The guard is observable *only* through this mutation: + // SniEndPoint#storeUnresolved normalizes a resolved proxy to the same string the constructor + // would have snapshotted, so any two endpoints built at the same instant match with or without + // it. What the snapshot buys is that they still match across a mutation of the caller's + // instance -- and provoking one needs a real reverse lookup, which needs 127.0.0.1 to have a + // mapping. Where it does not, this skips; a skipped run in the surefire report is the signal + // that this guard went unverified on that host, not that it passed. + assumeThat(proxy.getHostString()) + .as("requires a host where 127.0.0.1 starts out rendering as the literal") + .isEqualTo("127.0.0.1"); + + CloudTopologyMonitor monitor = new CloudTopologyMonitor(context, proxy); + UUID hostId = UUID.randomUUID(); + AdminRow row = Mockito.mock(AdminRow.class); + when(row.getUuid("host_id")).thenReturn(hostId); + EndPoint localEndPoint = Mockito.mock(EndPoint.class); + + EndPoint before = monitor.buildNodeEndPoint(row, null, localEndPoint); + + // What the SSL engine factory does to the caller's instance, behind the driver's back. + proxy.getHostName(); + assumeThat(proxy.getHostString()) + .as("requires a host where 127.0.0.1 reverse-resolves to a name") + .isNotEqualTo("127.0.0.1"); + + EndPoint after = monitor.buildNodeEndPoint(row, null, localEndPoint); + + assertThat(after).isEqualTo(before); + assertThat(after.hashCode()).isEqualTo(before.hashCode()); + assertThat(after.asMetricPrefix()).isEqualTo(before.asMetricPrefix()); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 7da8fb39415..28661200f31 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -19,7 +19,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import ch.qos.logback.classic.Level; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.util.LoggerTest; +import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; import org.junit.Test; @@ -57,4 +65,184 @@ public void should_reject_null_address() { .isInstanceOf(NullPointerException.class) .hasMessage("address can't be null"); } + + @Test + public void resolve_returns_already_resolved_address_as_is() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + InetSocketAddress resolved = endPoint.resolve(); + assertThat(resolved.isUnresolved()).isFalse(); + assertThat(resolved.getHostString()).isEqualTo("127.0.0.1"); + } + + @Test + public void resolve_passes_unresolved_hostname_through_without_looking_it_up() { + // This endpoint does NOT resolve hostnames itself. It hands the unresolved address to + // ChannelFactory, which expands it through Netty's AddressResolverGroup so that a custom + // resolver installed via NettyOptions#afterBootstrapInitialized still applies -- a direct + // InetAddress.getAllByName() call here would bypass it, and would block the admin event loop + // that connect() runs on. "localhost" would resolve fine, so this assertion is only meaningful + // because we check the address comes back *unresolved*. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + + InetSocketAddress resolved = endPoint.resolve(); + + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo("localhost"); + assertThat(resolved.getPort()).isEqualTo(9042); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_hostname() { + // No lookup happens, so an unresolvable name is not an error at this level: the connect attempt + // fails later with a descriptive error instead. + DefaultEndPoint endPoint = + new DefaultEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_override_resolution_but_preserve_identity() { + InetSocketAddress hostname = InetSocketAddress.createUnresolved("test.com", 9042); + DefaultEndPoint original = new DefaultEndPoint(hostname); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + // Resolution now yields exactly the pinned address... + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // ...but the copy still denotes the same node, and metric names must not change depending on + // which IP a connection happened to land on -- including through toString(), which is what + // TaggingMetricIdGenerator tags node metrics with, and nodes do adopt pinned copies. + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + // Equality has to hold in both directions: endpoints are used as set and map keys. + assertThat(original).isEqualTo(pinned); + // The original is untouched. + assertThat(original.resolve()).isEqualTo(hostname); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + DefaultEndPoint original = + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.com", 9042)); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((DefaultEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_return_same_instance_when_address_is_already_the_endpoints_own() { + // An already-resolved endpoint expands to exactly one candidate -- itself -- so ChannelFactory + // pins it to the address it already holds. That copy would be indistinguishable from the + // original in every respect, so there is no point allocating it. Every node discovered from the + // peers rows takes this path. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + DefaultEndPoint endPoint = new DefaultEndPoint(resolved); + + assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); + assertThat(endPoint.toString()).isEqualTo(resolved.toString()); + } + + @Test + public void pin_to_should_reject_null_address() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThatThrownBy(() -> endPoint.pinTo(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("resolvedAddress can't be null"); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // ChannelFactory pins whatever address it connected to; a non-Inet one (e.g. the local + // transport + // used by unit tests) cannot be held in an InetSocketAddress field, so pinning is skipped + // rather + // than failing the connection. + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolveCandidates() hands the original address straight through when the user disabled + // Netty's + // resolver, when the resolver does not support the address, or when it reports it as already + // resolved. Pinning a name would freeze resolve() on something that must re-expand on every + // connect -- no address stability gained, and the endpoint's own source silenced. The same + // guard + // is in SniEndPoint and ClientRoutesEndPoint. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("db.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void should_still_compare_an_unresolved_address_against_a_resolved_one() { + // Kept working for Metadata#findNode, whose caller may hold either form -- but it costs a DNS + // lookup on the calling thread, only compares the first address the name maps to, and does not + // agree with hashCode(). equals() warns once per JVM about it; that latch is static, so whether + // this test is the one that trips it depends on test ordering and is deliberately not asserted. + // See https://github.com/scylladb/java-driver/issues/1006. + DefaultEndPoint unresolved = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + DefaultEndPoint resolved = new DefaultEndPoint(new InetSocketAddress("localhost", 9042)); + + assertThat(unresolved).isEqualTo(resolved); + assertThat(resolved).isEqualTo(unresolved); + // ... while hashing differently, which is exactly the inconsistency the issue tracks. + assertThat(unresolved.hashCode()).isNotEqualTo(resolved.hashCode()); + } + + @Test + public void should_not_warn_when_comparing_an_ip_literal_against_a_resolved_address() { + // Contact points are stored unresolved whatever form they were written in -- AddressUtils + // .extract is always called with resolve = false -- so a plain "1.2.3.4:9042" meets a resolved + // peer endpoint on every comparison. That is the most ordinary deployment there is, and none of + // the three hazards the warning names applies to it: re-building an IP literal parses it with + // no resolver call, keeps the only address it can denote, and agrees with hashCode(). + DefaultEndPoint.LOGGED_MIXED_COMPARISON_WARNING.set(false); + LoggerTest.LoggerSetup logger = LoggerTest.setupTestLogger(DefaultEndPoint.class, Level.WARN); + try { + DefaultEndPoint literal = + new DefaultEndPoint(InetSocketAddress.createUnresolved("127.0.0.1", 9042)); + DefaultEndPoint resolved = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + + assertThat(literal).isEqualTo(resolved); + assertThat(resolved).isEqualTo(literal); + + verify(logger.appender, never()).doAppend(any()); + } finally { + logger.close(); + } + } + + @Test + public void should_warn_when_comparing_a_host_name_against_a_resolved_address() { + // The counterpart: a *name* on the unresolved side does cost a lookup on the calling thread and + // does only compare the first address it maps to, so the warning still has to fire there. + DefaultEndPoint.LOGGED_MIXED_COMPARISON_WARNING.set(false); + LoggerTest.LoggerSetup logger = LoggerTest.setupTestLogger(DefaultEndPoint.class, Level.WARN); + try { + DefaultEndPoint name = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + DefaultEndPoint resolved = new DefaultEndPoint(new InetSocketAddress("localhost", 9042)); + + assertThat(name).isEqualTo(resolved); + + verify(logger.appender, times(1)).doAppend(logger.loggingEventCaptor.capture()); + assertThat(logger.loggingEventCaptor.getValue().getFormattedMessage()) + .contains("Compared an unresolved host name against a resolved endpoint address"); + } finally { + logger.close(); + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java index 6a53fe3e433..f844f9d2805 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java @@ -18,10 +18,31 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.codahale.metrics.MetricRegistry; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfig; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; +import com.datastax.oss.driver.api.core.metrics.NodeMetric; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.internal.core.metrics.AbstractMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.DefaultMetricIdGenerator; +import com.datastax.oss.driver.internal.core.metrics.DropwizardNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Collections; +import java.util.Set; import java.util.UUID; import org.junit.Test; @@ -55,4 +76,239 @@ public void should_have_expected_string_representation_if_hostid_is_null() { "Node(endPoint=localhost/127.0.0.1:9042, hostId=null, hashCode=%x)", node.hashCode()); assertThat(node.toString()).isEqualTo(expected); } + + @Test + public void should_adopt_a_newer_endpoint_that_only_differs_by_its_pinned_address() { + // A PinnableEndPoint copy compares equal to the original -- pinnedAddress is excluded from + // equals() by contract, so that a pinned copy still denotes the same node. setEndPoint() must + // therefore not use equals() to decide whether to adopt it: the pinned address is the one every + // subsequent connection to this node will use, so refusing the newer instance would freeze the + // node on the first address it ever connected to, even after the control connection has moved + // and told us about it. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + DefaultNode node = new DefaultNode(endPoint, context); + + EndPoint pinnedToFirst = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)); + node.setEndPoint(pinnedToFirst, context); + assertThat(node.getEndPoint()).isSameAs(pinnedToFirst); + + EndPoint pinnedToSecond = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.3", 9042)); + // Same node by equals(), different pinned address. + assertThat(pinnedToSecond).isEqualTo(pinnedToFirst); + node.setEndPoint(pinnedToSecond, context); + + assertThat(node.getEndPoint()).isSameAs(pinnedToSecond); + assertThat(node.getEndPoint().resolve()).isEqualTo(new InetSocketAddress("127.0.0.3", 9042)); + } + + @Test + public void should_keep_the_endpoint_instance_it_already_holds_when_nothing_would_differ() { + // Every full topology refresh mints a fresh endpoint over a fresh InetSocketAddress for every + // node, and for an unchanged node it denotes exactly what this one already does. Adopting it + // would discard the reverse-DNS name that an earlier TLS handshake cached on the InetAddress + // this + // node holds (DefaultSslEngineFactory calls getHostName() under the default + // allow-dns-reverse-lookup-san = true), so the next connection would repeat that blocking + // lookup + // on an I/O event loop -- once per node per refresh instead of once per node per session. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint held = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(held, context); + + EndPoint equivalent = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(equivalent).isNotSameAs(held); + node.setEndPoint(equivalent, context); + + assertThat(node.getEndPoint()).isSameAs(held); + } + + @Test + public void should_adopt_an_endpoint_of_a_different_kind_that_resolves_to_the_same_address() { + // A dynamic endpoint (ClientRoutesEndPoint) resolves through its topology monitor on every + // call, + // so it is not interchangeable with a static one that happens to point at the same address + // today. + // The early return must not keep the static one in that case. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint stat1c = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(stat1c, context); + + EndPoint dynamic = new RenamingEndPoint("/127.0.0.1:9042"); + assertThat(dynamic.resolve()).isEqualTo(stat1c.resolve()); + assertThat(dynamic.asMetricPrefix()).isEqualTo(stat1c.asMetricPrefix()); + + node.setEndPoint(dynamic, context); + + assertThat(node.getEndPoint()).isSameAs(dynamic); + } + + @Test + public void should_not_rebuild_the_metric_updater_for_a_pin_only_change() { + // A pinned copy is identified exactly like the original -- same asMetricPrefix(), same + // toString() -- so rebuilding would clear and re-register metrics under identical names, and + // reset their values along the way. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + DefaultNode node = new DefaultNode(endPoint, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint( + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)), context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + @Test + public void should_not_rebuild_the_metric_updater_when_only_the_endpoints_string_form_differs() { + // toString() is not usable as an identity key because it is not stable across instances that + // denote the same thing: DefaultEndPoint delegates it to InetSocketAddress, which renders + // InetAddress's cached hostName field, and that field is filled in the first time anything + // calls getHostName() -- which DefaultSslEngineFactory does, on this node's own endpoint + // instance, under the default allow-dns-reverse-lookup-san = true. Keying on it would clear and + // re-register every node's metrics on every topology refresh, since each refresh decodes a + // fresh endpoint whose hostName has not been filled in yet. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint beforeLookup = new RenamingEndPoint("/127.0.0.1:9042"); + EndPoint afterLookup = new RenamingEndPoint("host.example.com/127.0.0.1:9042"); + assertThat(beforeLookup.asMetricPrefix()).isEqualTo(afterLookup.asMetricPrefix()); + assertThat(beforeLookup.toString()).isNotEqualTo(afterLookup.toString()); + + DefaultNode node = new DefaultNode(beforeLookup, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(afterLookup, context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + /** An endpoint that always reports the same metric prefix but renders differently. */ + private static class RenamingEndPoint implements EndPoint { + private final String stringForm; + + RenamingEndPoint(String stringForm) { + this.stringForm = stringForm; + } + + @NonNull + @Override + public SocketAddress resolve() { + return new InetSocketAddress("127.0.0.1", 9042); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "127_0_0_1:9042"; + } + + @Override + public String toString() { + return stringForm; + } + } + + @Test + public void should_rebuild_the_metric_updater_when_an_equal_endpoint_renames_the_metrics() { + // An unresolved hostname and the address it resolves to compare *equal* (see + // DefaultEndPoint#equals) but do not produce the same metric prefix. That is exactly what + // happens when a contact-point node adopts the endpoint built from its system.local row, so + // deciding on equals() alone would leave the node's metrics registered under the hostname while + // asMetricPrefix() had moved on to the IP. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(asHostname).isEqualTo(asAddress); + assertThat(asHostname.asMetricPrefix()).isNotEqualTo(asAddress.asMetricPrefix()); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(asAddress, context); + + assertThat(node.getMetricUpdater()).isSameAs(second); + verify(first).clearMetrics(); + } + + @Test + public void should_rebuild_the_metric_updater_without_wiping_the_metrics_it_registers() { + // The two tests above use mocks, which cannot see *when* clearMetrics() runs relative to the + // endpoint swap -- and that order is what decides whether the rebuild works. Dropwizard and + // MicroProfile do not remember the ids they registered under: clearMetrics() recomputes each + // one + // from the node's endpoint as it stands at that moment. Clearing after the swap therefore + // removes exactly the series the new updater has just registered, and leaves the old ones in + // the + // registry with nothing writing to them. So this one drives a real registry. + MetricRegistry registry = new MetricRegistry(); + NodeMetric metric = DefaultNodeMetric.UNSENT_REQUESTS; + InternalDriverContext context = dropwizardContext(registry, Collections.singleton(metric)); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + String underHostname = "s.nodes." + asHostname.asMetricPrefix() + '.' + metric.getPath(); + String underAddress = "s.nodes." + asAddress.asMetricPrefix() + '.' + metric.getPath(); + assertThat(underHostname).isNotEqualTo(underAddress); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(registry.getNames()).containsExactly(underHostname); + + node.setEndPoint(asAddress, context); + + assertThat(registry.getNames()).containsExactly(underAddress); + } + + /** A context wired to a real Dropwizard registry, enough for {@link DefaultNode} to use it. */ + private static InternalDriverContext dropwizardContext( + MetricRegistry registry, Set enabledMetrics) { + InternalDriverContext context = mock(InternalDriverContext.class); + DriverConfig config = mock(DriverConfig.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + when(context.getSessionName()).thenReturn("s"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(profile.getString(DefaultDriverOption.METRICS_ID_GENERATOR_PREFIX, "")).thenReturn(""); + // Built outside the when(): the generator's constructor reads back from the context, and + // Mockito rejects a nested call on a stubbing that is still open. + DefaultMetricIdGenerator idGenerator = new DefaultMetricIdGenerator(context); + when(context.getMetricIdGenerator()).thenReturn(idGenerator); + + MetricsFactory metricsFactory = mock(MetricsFactory.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + // Built on demand rather than up front: an updater registers its metrics from its constructor, + // under the names the node's endpoint yields at that point. + when(metricsFactory.newNodeUpdater(any())) + .thenAnswer( + invocation -> + new DropwizardNodeMetricUpdater( + invocation.getArgument(0), context, enabledMetrics, registry)); + return context; + } + + /** A context whose only stubbed behaviour is the metrics factory {@code DefaultNode} asks for. */ + private static InternalDriverContext contextWith(MetricsFactory metricsFactory) { + InternalDriverContext context = mock(InternalDriverContext.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + return context; + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java index 892715a47c3..3f95823fab0 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; import com.datastax.oss.driver.internal.core.addresstranslation.PassThroughAddressTranslator; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; @@ -443,6 +444,123 @@ public void should_fail_get_channel_node_info_if_local_result_is_empty() { }); } + @Test + public void should_query_system_local_for_channel_node_info() { + // Given — a channel to identify (the control connection's connect hook is the caller). + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getHostId()).isEqualTo(hostId)); + } + + @Test + public void should_identify_the_connected_node_by_the_address_it_reached_not_the_contact_point() + throws Exception { + // Given — a control channel that came up through a contact-point hostname. ChannelFactory binds + // the endpoint it hands the channel to the one address that connection reached, and by + // PinnableEndPoint's contract that copy is identified exactly like the unpinned original. + // + // The pinned address carries the queried *name*, which is what the connect path really + // produces: + // the JDK and Netty-DNS resolvers label their results with it, and ChannelFactory + // #reattachHostname restores it when a custom resolver does not. A bare + // new InetSocketAddress("127.0.0.1", 9042) would be a shape that path cannot yield, and would + // let this test pass against an implementation that reads the host string back. + EndPoint contactPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + InetSocketAddress reached = + new InetSocketAddress( + InetAddress.getByAddress("db.example.com", new byte[] {127, 0, 0, 1}), 9042); + EndPoint pinned = ((PinnableEndPoint) contactPoint).pinTo(reached); + assertThat(pinned.asMetricPrefix()).isEqualTo("db_example_com:9042"); + assertThat(((InetSocketAddress) pinned.resolve()).getHostString()).isEqualTo("db.example.com"); + when(channel.getEndPoint()).thenReturn(pinned); + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then — the node is identified by its own address, not by the name it was reached through. + // Keeping the contact point's identity would not even be exclusively this node's: the + // reconnection fallback hands the contact points back every round, so each successive control + // node would take the same metric prefix, and two live nodes sharing one prefix means the older + // one's clearMetrics() deletes the newcomer's series. + assertThatStage(futureNodeInfo) + .isSuccess( + nodeInfo -> { + assertThat(nodeInfo.getEndPoint().asMetricPrefix()).isEqualTo("127_0_0_1:9042"); + // What the node connects to is unchanged, label included, so the TLS peer host and + // the Kerberos service name stay the name the operator configured -- with no reverse + // lookup. + InetSocketAddress resolved = (InetSocketAddress) nodeInfo.getEndPoint().resolve(); + assertThat(resolved).isEqualTo(reached); + assertThat(resolved.getHostString()).isEqualTo("db.example.com"); + assertThat(resolved.getAddress().getHostName()).isEqualTo("db.example.com"); + }); + } + + @Test + public void should_not_let_a_cached_reverse_name_change_the_connected_nodes_identity() + throws Exception { + // An IP-literal contact point. Its pinned address starts out nameless, and begins reporting a + // reverse-DNS name as soon as DefaultSslEngineFactory calls getHostName() on the shared + // InetAddress -- which is what an already-labelled instance stands in for here. The identity + // has + // to come from the bytes, or it would depend on whether TLS is enabled and whether a PTR record + // exists, and would move mid-session for the same node. + EndPoint contactPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("127.0.0.1", 9042)); + InetSocketAddress reachedWithPtrName = + new InetSocketAddress( + InetAddress.getByAddress("node1.internal.example.com", new byte[] {127, 0, 0, 1}), + 9042); + EndPoint pinned = ((PinnableEndPoint) contactPoint).pinTo(reachedWithPtrName); + when(channel.getEndPoint()).thenReturn(pinned); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", + mockResult(mockLocalRow(1, UUID.randomUUID())))); + + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // The identity is already the literal, so there is nothing to rebuild and the instance is kept. + assertThatStage(futureNodeInfo) + .isSuccess( + nodeInfo -> { + assertThat(nodeInfo.getEndPoint()).isSameAs(pinned); + assertThat(nodeInfo.getEndPoint().asMetricPrefix()).isEqualTo("127_0_0_1:9042"); + }); + } + + @Test + public void should_keep_the_channels_own_endpoint_when_it_already_names_the_connected_address() { + // The reconnection-to-a-known-node case, and every refresh after the first: nothing was pinned + // because the endpoint already held the address, so the existing instance is kept -- which is + // what keeps the control-node check in refreshNode() an identity comparison rather than an + // endpoint equals() (and DefaultEndPoint's equals resolves names). + EndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + when(channel.getEndPoint()).thenReturn(endPoint); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", + mockResult(mockLocalRow(1, UUID.randomUUID())))); + + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getEndPoint()).isSameAs(endPoint)); + } + @Test public void should_stop_executing_queries_once_closed() { // Given @@ -651,6 +769,47 @@ public void should_use_projected_query_on_second_refresh_node_list_call() { assertThatStage(futureInfos).isSuccess(infos -> assertThat(infos).hasSize(2)); } + @Test + public void should_relearn_the_local_projection_from_every_response() { + // The control connection clears this cache before each candidate's identity read -- but that + // orders the reads, not the answers. A candidate abandoned on the connect-hook timeout is + // abandoned rather than cancelled, so both reads can be outstanding at once and the refused + // one can answer first. A cache that filled only when empty would take its columns and then + // decline the accepted candidate's, and since the projection is an intersection that can only + // shrink, the node the driver keeps would stop reporting every column the refused one lacked -- + // dse_version here, and with it the driver's whole idea of what server it is talking to. + UUID hostId = node1.getHostId(); + ImmutableList narrow = ImmutableList.of("rpc_address", "data_center", "host_id"); + ImmutableList wide = + ImmutableList.of("rpc_address", "data_center", "host_id", "dse_version"); + CompletableFuture refusedAnswer = new CompletableFuture<>(); + CompletableFuture keptAnswer = new CompletableFuture<>(); + topologyMonitor.stubQueries( + new StubbedQuery("SELECT * FROM system.local WHERE key='local'", refusedAnswer), + new StubbedQuery("SELECT * FROM system.local WHERE key='local'", keptAnswer), + new StubbedQuery( + "SELECT " + String.join(", ", wide) + " FROM system.local WHERE key='local'", + AdminResultTestHelper.mockResultWithColumns(wide, mockLocalRow(1, hostId)))); + + // When -- both reads go out with the cache cleared, as the hook does for two candidates, and + // the one the loop refused is the first to answer. + topologyMonitor.resetLocalColumnCache(); + CompletionStage refused = topologyMonitor.getChannelNodeInfo(channel); + topologyMonitor.resetLocalColumnCache(); + CompletionStage kept = topologyMonitor.getChannelNodeInfo(channel); + refusedAnswer.complete( + AdminResultTestHelper.mockResultWithColumns(narrow, mockLocalRow(1, hostId))); + keptAnswer.complete(AdminResultTestHelper.mockResultWithColumns(wide, mockLocalRow(1, hostId))); + assertThatStage(refused).isSuccess(); + assertThatStage(kept).isSuccess(); + + // Then -- a third read, with nothing cleared in front of it, projects the columns of the + // response that came *last*, not of the one that came first. The stub is the assertion: + // fill-only-when-empty would send `narrow` here and the query strings would not match. + assertThatStage(topologyMonitor.getChannelNodeInfo(channel)) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getHostId()).isEqualTo(hostId)); + } + @Test public void should_revert_to_select_star_after_reset_column_caches() { // Given — warm the caches with a first call @@ -768,7 +927,12 @@ protected CompletionStage query( "Unknown keyspace/cf pair (system.peers_v2)"); return CompletableFutures.failedFuture(new UnexpectedResponseException(queryString, error)); } - return CompletableFuture.completedFuture(nextQuery.result); + // A stub may hand back a future the test completes itself, which is the only way to have two + // of these reads outstanding at once -- the shape the connect hook produces when a candidate + // is abandoned on its timeout and answers anyway. + return (nextQuery.pending != null) + ? nextQuery.pending + : CompletableFuture.completedFuture(nextQuery.result); } } @@ -777,13 +941,29 @@ private static class StubbedQuery { private final Map parameters; private final AdminResult result; private final boolean error; + private final CompletableFuture pending; private StubbedQuery( - String queryString, Map parameters, AdminResult result, boolean error) { + String queryString, + Map parameters, + AdminResult result, + boolean error, + CompletableFuture pending) { this.queryString = queryString; this.parameters = parameters; this.result = result; this.error = error; + this.pending = pending; + } + + private StubbedQuery( + String queryString, Map parameters, AdminResult result, boolean error) { + this(queryString, parameters, result, error, null); + } + + /** A query whose response the test releases, so that reads can be left in flight. */ + private StubbedQuery(String queryString, CompletableFuture pending) { + this(queryString, Collections.emptyMap(), null, false, pending); } private StubbedQuery(String queryString, Map parameters, AdminResult result) { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java index 89b36b9ee09..d3caac0fdc1 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java @@ -33,17 +33,17 @@ import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy.DistanceReporter; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; -import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Metadata; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.context.EventBus; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; -import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.Map; import java.util.Objects; import java.util.Queue; @@ -79,6 +79,7 @@ public class LoadBalancingPolicyWrapperTest { private EventBus eventBus; @Mock private MetadataManager metadataManager; @Mock private Metadata metadata; + @Mock private TopologyMonitor topologyMonitor; @Mock protected MetricsFactory metricsFactory; @Captor private ArgumentCaptor> initNodesCaptor; @@ -102,13 +103,16 @@ public void setup() { when(metadata.getNodes()).thenReturn(allNodes); when(metadataManager.getContactPoints()).thenReturn(contactPoints); when(context.getMetadataManager()).thenReturn(metadataManager); + when(context.getTopologyMonitor()).thenReturn(topologyMonitor); when(context.getConfig()).thenReturn(config); when(config.getDefaultProfile()).thenReturn(defaultProfile); when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(false); - defaultPolicyQueryPlan = Lists.newLinkedList(ImmutableList.of(node3, node2, node1)); + // Use a real built-in QueryPlan (not a mutable LinkedList): its add()/addAll() throw, so the + // control-reconnection plan must compose rather than mutate it (see CompositeQueryPlan usage). + defaultPolicyQueryPlan = new SimpleQueryPlan(node3, node2, node1); when(policy1.newQueryPlan(null, null)).thenReturn(defaultPolicyQueryPlan); eventBus = spy(new EventBus("test")); @@ -130,26 +134,28 @@ public void setup() { @Test public void should_build_control_connection_query_plan_from_contact_points_before_init() { - // When + // When — before init, the control-reconnection plan is built straight from the contact points + // (bypassing the load balancing policies), so each hostname can be tried on the first connect. Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test public void should_build_query_plan_from_contact_points_before_init() { - // When + // When — before init, the query plan is built straight from the contact points (bypassing the + // load balancing policies) Queue queryPlan = wrapper.newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test @@ -204,17 +210,112 @@ public void should_fetch_control_connection_query_plan_from_policy_after_init() assertThat(queryPlan.poll()).isEqualTo(node3); assertThat(queryPlan.poll()).isEqualTo(node2); assertThat(queryPlan.poll()).isEqualTo(node1); - // Remaining nodes are contact points appended at the end. - // They are new DefaultNode instances created via newContactPoint, so compare by endpoint. - Set remainingEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - remainingEndpoints.add(n.getEndPoint()); + // Remaining nodes are the original contact points appended at the end. DefaultNode does not + // override equals, so comparing against the retained instances is an identity check. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); + } + + @Test + public void should_reuse_the_retained_contact_point_nodes_rather_than_minting_copies() { + // Given — the flag now defaults to true, so this plan is built on every control-connection + // reconnection round for every user, not only for those who opted in. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When — two rounds, as a reconnection sequence would produce. + Queue firstPlan = wrapper.newControlReconnectionQueryPlan(); + Queue secondPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — both plans hand out the very objects MetadataManager retains, not per-plan copies. + // A copy would be invisible to the metric updater registered for the real node and would fire + // its own controlConnectionFailed event, once per contact point per round, for as long as + // reconnection lasts. + // + // The size is asserted first, deliberately: iterating an empty plan would satisfy the loops + // below without checking anything, so a regression that stopped appending the fallback at all + // would pass. DefaultNode does not override equals, so containment is an identity check. + assertThat(firstPlan).hasSize(2); + assertThat(secondPlan).hasSize(2); + for (Node node : firstPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); + for (Node node : secondPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - assertThat(remainingEndpoints).isEqualTo(contactEndpoints); + // And the retained set itself is untouched by the shuffle. + assertThat(contactPoints).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void should_not_duplicate_contact_points_before_init() { + // Given — the wrapper hasn't been init()-ed yet (state=BEFORE_INIT), so newQueryPlan() already + // builds the regular plan directly from the contact points. The reconnect-contact-points flag + // doesn't matter here: newControlReconnectionQueryPlan() short-circuits on state before even + // reading it, since appending contact points again pre-init would just duplicate every entry in + // the plan. + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the contact points are read only once (not once for the regular plan and again for a + // redundant "fallback" append), and the plan has no duplicate entries. + verify(metadataManager, times(1)).getContactPoints(); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_reconnect_contact_points_is_disabled() { + // Given — the flag defaults to false in the test setup (see @Before) + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Only the policy query plan is returned; no contact points are appended. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_topology_monitor_reresolves_addresses() { + // Given — the flag is enabled, but the topology monitor re-resolves node addresses on its own + // (e.g. a proxy-based monitor such as client routes or the cloud SNI proxy). + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Contact points must not be appended: the monitor keeps addresses fresh, and appending raw + // contact points could resurrect nodes it has authoritatively removed. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_append_contact_points_when_query_plan_empty_even_if_topology_monitor_reresolves() { + // Given — the flag is enabled and the topology monitor re-resolves node addresses on its own, + // but the live-node query plan is empty. With no node to try, reconnection can only recover + // through the contact-point fallback, so it must be appended despite the re-resolving monitor. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the retained contact-point instances are appended, in some order. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test @@ -223,24 +324,15 @@ public void should_return_contact_points_when_query_plan_empty_and_flag_enabled( when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(true); wrapper.init(); - // Make the policy return an empty query plan - when(policy1.newQueryPlan(null, null)).thenReturn(Lists.newLinkedList(ImmutableList.of())); + // Make the policy return an empty query plan (QueryPlan.EMPTY, as the real policies do) + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); // When Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); // Then - // Should get the contact points (compare by endpoint since they are new instances) - assertThat(queryPlan.size()).isEqualTo(contactPoints.size()); - Set resultEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - resultEndpoints.add(n.getEndPoint()); - } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); - } - assertThat(resultEndpoints).isEqualTo(contactEndpoints); + // Should get the retained contact-point instances themselves. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java new file mode 100644 index 00000000000..6f26cb6bae7 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class SniEndPointTest { + + @Test + public void resolve_returns_the_proxy_address_as_is_without_looking_it_up() { + // The proxy address is a hostname (that is how CloudConfigFactory builds it) and this endpoint + // must not resolve it: ChannelFactory expands it through Netty's AddressResolverGroup, which is + // what makes a custom resolver apply to the SNI proxy too, and what keeps resolve() safe to + // call + // from an event loop -- SniSslEngineFactory#newSslEngine does exactly that. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + SniEndPoint endPoint = new SniEndPoint(proxy, "test-server-name"); + + assertThat(endPoint.resolve()).isSameAs(proxy); + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + } + + @Test + public void should_keep_a_resolved_proxy_hostname_unresolved() { + // InetSocketAddress(String, int) resolves eagerly, so a hostname passed to + // withCloudProxyAddress() arrives here already bound to one of its IPs. Storing it that way + // would freeze every Cloud connection on that IP for the life of the session: resolve() hands + // the stored address straight to the connection layer, which only expands unresolved ones. + InetSocketAddress resolvedProxy = new InetSocketAddress("localhost", 9042); + assertThat(resolvedProxy.isUnresolved()).isFalse(); + + SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("localhost"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + // Normalization is unconditional, so endpoints built from either form of the same proxy still + // denote the same node -- equals() keys on the stored address. + assertThat(endPoint) + .isEqualTo( + new SniEndPoint( + InetSocketAddress.createUnresolved("localhost", 9042), "test-server-name")); + // The metric prefix is unaffected either way: it was already built from the host string. + assertThat(endPoint.asMetricPrefix()).isEqualTo("localhost:9042_test-server-name"); + } + + @Test + public void should_store_a_proxy_given_as_an_ip_address_unresolved_too() { + // An IP literal has nothing to expand, but it is stored unresolved all the same, because that + // is what makes this endpoint's identity immune to drift: see the test below. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("127.0.0.1"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void identity_should_not_drift_when_the_source_address_grows_a_reverse_dns_name() { + // SniSslEngineFactory#newSslEngine calls getHostName() on the address resolve() hands it, under + // the default allow-dns-reverse-lookup-san = true. On a *resolved* address that caches the + // reverse-DNS name on the underlying InetAddress, and getHostString() reports it from then on. + // An endpoint keyed on such an instance would silently change its metric prefix mid-session, + // moving a node's metrics; storing the address unresolved removes the field that changes. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + InetSocketAddress storedBefore = endPoint.resolve(); + + // What the SSL engine factory does. 127.0.0.1 has a PTR record on essentially every machine, so + // this really does change the source instance's host string. Assumed rather than asserted -- + // it is this test's precondition, not its subject, and a host without a loopback PTR entry (a + // minimal container image, nsswitch without `files`) would otherwise report SniEndPoint as + // broken over an environment fact. Same treatment as CloudTopologyMonitorTest and + // AddressUtilsTest give it. + ipProxy.getHostName(); + assumeThat(ipProxy.getHostString()) + .as("127.0.0.1 has no reverse-DNS name here, so the drift this test guards cannot occur") + .isNotEqualTo("127.0.0.1"); + + assertThat(endPoint.resolve()).isEqualTo(storedBefore); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + assertThat(endPoint).isEqualTo(new SniEndPoint(storedBefore, "test-server-name")); + + // And the driver no longer pollutes the address it holds in the first place: getHostName() on + // an + // unresolved address returns its stored host string without looking anything up, so the call + // above cannot happen to *this* instance. (What the SSL engine actually sees is the pinned + // copy, + // whose address is excluded from equals() and asMetricPrefix() by contract.) + assertThat(endPoint.resolve().getHostName()).isEqualTo("127.0.0.1"); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_proxy_hostname() { + // No lookup happens here, so an unresolvable name only fails later, at connect time. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042), + "test-server-name"); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_make_resolve_return_the_connected_proxy_ip_and_preserve_identity() { + // Pinning is what lets SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel + // initializer -- see the very proxy IP the channel is connected to, rather than the hostname or + // another A-record. + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolve().isUnresolved()).isFalse(); + // The original is untouched. + assertThat(original.resolve().isUnresolved()).isTrue(); + + // The pinned copy still denotes the same node, down to every string it is identified by: the + // tagging MetricIdGenerator tags node metrics with the endpoint's toString(), and nodes do + // adopt + // pinned copies. + assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // ChannelFactory hands the address straight back when the user disabled Netty's resolver or a + // custom one declines it. Pinning that would freeze resolve() on a name that must re-expand on + // every connect, silencing the proxy's A-record fallback for good. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("proxy.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(new LocalAddress("test"))).isSameAs(endPoint); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java index ccc42a7027d..31a39df33d8 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java @@ -18,8 +18,12 @@ package com.datastax.oss.driver.internal.core.metrics; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,16 +37,22 @@ import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.NodeMetric; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.util.LoggerTest; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; import java.time.Duration; import java.util.Collections; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; @RunWith(DataProviderRunner.class) public class DropwizardNodeMetricUpdaterTest { @@ -152,6 +162,359 @@ protected void initializeHdrTimer( verify(logger.appender, timeout(500).times(0)).doAppend(logger.loggingEventCaptor.capture()); } + @Test + public void should_adopt_a_pending_expiration_from_the_updater_it_replaces() { + // given – two updaters for the same node, as DefaultNode.setEndPoint builds when an endpoint + // change renames the metrics, and an expiration already armed on the one being replaced. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout pendingTimeout = mock(Timeout.class); + Timeout adoptedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(pendingTimeout, adoptedTimeout); + // Still pending, i.e. the countdown had not run yet: that is what makes it worth carrying over. + when(pendingTimeout.cancel()).thenReturn(true); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // when + replacement.adoptExpirationFrom(previous); + + // then – the countdown is cancelled on the object nothing refers to any more and re-armed on + // the + // one whose metrics it will clear. Without this the expiration is simply lost: it is armed and + // cancelled through node.getMetricUpdater(), so a node that is already down never produces + // another DOWN event to arm the replacement, while the orphan timer still fires clearMetrics() + // on names recomputed from whatever endpoint the node holds by then. + verify(pendingTimeout).cancel(); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // and – there is nothing left to hand over a second time. + replacement.adoptExpirationFrom(previous); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_re_arm_an_expiration_that_has_already_fired() throws Exception { + // The countdown ran, so this node's metrics were cleared. If its endpoint then changes -- a new + // address in the peers row, a client route published -- DefaultNode#setEndPoint rebuilds the + // updater, and the new one's constructor eagerly re-registers the node's whole metric set. So + // the series are back, for a node that is still down. + // + // Nothing else would arm a countdown for them: startMetricsExpirationTimeout()'s only other + // caller is the metrics factory's DOWN/FORCED_DOWN/removed handler, and a node that is already + // down produces no such event. Arming nothing here leaves the resurrected series to outlive the + // node until it next comes up or is removed, which may be never. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout firedTimeout = mock(Timeout.class); + Timeout adoptedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(firedTimeout, adoptedTimeout); + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(TimerTask.class); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(taskCaptor.capture(), anyLong(), any(TimeUnit.class)); + // Run the countdown to completion, which is what leaves the updater with no pending timeout and + // its metrics cleared -- the state the cancel() return value alone cannot distinguish from a + // node that simply never had one armed. + taskCaptor.getValue().run(firedTimeout); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_re_arm_an_expiration_that_is_still_running() throws Exception { + // The gap between "fired" and "recorded as fired". Netty flips a HashedWheelTimeout to + // ST_EXPIRED before it runs the task, and the task clears a whole metric set before it reaches + // its compare-and-set -- so for that whole interval cancel() answers false while the reference + // still holds the real timeout, exactly as it would for a timeout somebody else already + // cancelled. Reading cancel() to decide whether there is a countdown to carry therefore drops + // the one this method exists for, and the task's own compare-and-set then loses against the + // getAndSet(null) the adoption just did -- so neither side arms anything, and a still-down + // node keeps the metric set its replacement's constructor re-registered, for good. + // + // Non-null is the answer instead: the reference is cleared only when nothing is armed. + // + // This input used to be asserted the other way round, by a test reading a failing cancel() as + // "already cancelled, nothing to carry". Netty returns false for ST_CANCELLED and ST_EXPIRED + // alike, so cancel() cannot tell those apart -- and only one of them can be in the reference: + // cancelMetricsExpirationTimeout() clears it *before* cancelling, and the arming accumulator + // only ever cancels a candidate it declined to store. A non-null reference whose cancel() fails + // is therefore an expiry in flight, never a spent one. The genuine "nothing to carry" case is a + // cleared reference, which should_not_re_arm_an_expiration_on_a_node_that_came_back_up covers. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout firing = mock(Timeout.class); + Timeout adopted = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(firing, adopted); + // What Netty answers once the task is on its way: too late to cancel, not yet recorded. + when(firing.cancel()).thenReturn(false); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_not_cancel_a_timeout_from_inside_the_arming_accumulator() { + // AtomicReference re-applies its accumulator when the compare-and-set loses, so the function + // has to be pure. It used to cancel the candidate it declined, and the re-application is + // precisely where that bites: a first pass that saw a live countdown, cancelled the newcomer + // and kept the old one gets re-run against a reference the timer task has since set to EXPIRED + // -- and now returns the newcomer it just cancelled. Storing that leaves a handle that never + // fires and is neither null nor EXPIRED, so every later arm keeps it and cancels the fresh one + // instead, and the node's metrics stop expiring for as long as it stays down. + // + // Asserted on the accumulator rather than through startMetricsExpirationTimeout(), because a + // caller cannot show this without faking the contention: what is wrong is the shape of the + // function, and that is what this pins. + Timeout armed = mock(Timeout.class); + Timeout candidate = mock(Timeout.class); + + assertThat(AbstractMetricUpdater.keepArmed(armed, candidate)).isSameAs(armed); + assertThat(AbstractMetricUpdater.keepArmed(null, candidate)).isSameAs(candidate); + assertThat(AbstractMetricUpdater.keepArmed(AbstractMetricUpdater.EXPIRED, candidate)) + .isSameAs(candidate); + + verify(candidate, never()).cancel(); + verify(armed, never()).cancel(); + } + + @Test + public void should_not_report_an_expiry_a_cancel_won() throws Exception { + // The interleaving that used to leave a healthy node's metrics on a death clock. The expiry + // task was clearMetrics(), then cancelMetricsExpirationTimeout() -- which itself cleared the + // "expired" flag -- then expired.set(true). A cancel from the metrics factory's UP handler + // landing between the last two left the flag latched on a node that was live again, and the + // next endpoint change then handed adoptExpirationFrom a countdown to re-arm: an expire-after + // later, clearMetrics() wiped the whole series of a healthy node, with nothing to re-register + // it. + // + // Both halves are now one compare-and-set on the timeout reference, so whichever of the two + // arrives second finds the other already there. Asserted by driving the interleaving rather + // than racing for it: a race cannot demonstrate the absence of a window. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout firedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(firedTimeout); + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(TimerTask.class); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + + // The node goes down and the countdown is armed. + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(taskCaptor.capture(), anyLong(), any(TimeUnit.class)); + + // It fires, and the node comes back up while the task is between its two statements. Netty + // marks a timeout ST_EXPIRED before invoking its task, so this cancel() is already failing. + previous.cancelMetricsExpirationTimeout(); + taskCaptor.getValue().run(firedTimeout); + + // The node is live, so its endpoint changing must not arm anything. + replacement.adoptExpirationFrom(previous); + + verify(timer, times(1)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_report_an_expiry_no_cancel_intervened_in() throws Exception { + // The direction that must keep working: a node whose metrics really did expire while it was + // down, and whose endpoint then changed. The replacement's constructor re-registers the whole + // series, so without a countdown those resurrected metrics would outlive the node until it + // next came up or was removed -- which may be never, since the factory's own + // startMetricsExpirationTimeout() only runs on a DOWN event a node that is already down does + // not produce. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout firedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(firedTimeout); + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(TimerTask.class); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(taskCaptor.capture(), anyLong(), any(TimeUnit.class)); + taskCaptor.getValue().run(firedTimeout); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_not_re_arm_an_expiration_on_a_node_that_came_back_up() { + // The other side of the same coin: cancelMetricsExpirationTimeout() is what the metrics factory + // calls on UP/UNKNOWN, so a node that is live again has nothing to hand over. Arming a + // countdown + // for it would clear a healthy node's metrics an expire-after later. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout cancelledTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(cancelledTimeout); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + previous.cancelMetricsExpirationTimeout(); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(1)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_not_fail_the_caller_when_the_timer_is_already_stopped() { + // adoptExpirationFrom is reached from DefaultNode#setEndPoint, i.e. from NodesRefresh#copyInfos + // inside MetadataManager's apply step, which neither catches nor contains throwables. And + // HashedWheelTimer.newTimeout throws once the timer has been stopped (NettyOptions#onClose) or + // its pending-task ceiling is reached. Letting that out would drop an entire metadata refresh + // -- + // surfacing only as a DEBUG line -- over an hour-scale cleanup countdown. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout pendingTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(pendingTimeout); + when(pendingTimeout.cancel()).thenReturn(true); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + + // The session is closing: the timer refuses new tasks from here on. + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenThrow(new IllegalStateException("cannot be started once stopped")); + + // Then — the handover is abandoned quietly rather than propagating. + replacement.adoptExpirationFrom(previous); + } + + /** A node updater with metric registration stubbed out: only its expiration behavior matters. */ + private static DropwizardNodeMetricUpdater newNodeUpdater( + Node node, InternalDriverContext context) { + return new DropwizardNodeMetricUpdater( + node, + context, + Collections.singleton(DefaultNodeMetric.CQL_MESSAGES), + new MetricRegistry()) { + @Override + protected void initializeGauge( + NodeMetric metric, DriverExecutionProfile profile, Supplier supplier) { + // do nothing + } + + @Override + protected void initializeCounter(NodeMetric metric, DriverExecutionProfile profile) { + // do nothing + } + + @Override + protected void initializeHdrTimer( + NodeMetric metric, + DriverExecutionProfile profile, + DriverOption highestLatency, + DriverOption significantDigits, + DriverOption interval) { + // do nothing + } + }; + } + @DataProvider public static Object[][] acceptableEvictionTimes() { return new Object[][] { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/pool/ChannelPoolInitTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/pool/ChannelPoolInitTest.java index d4fb0211173..96e73814cf2 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/pool/ChannelPoolInitTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/pool/ChannelPoolInitTest.java @@ -31,7 +31,14 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; import com.datastax.oss.driver.api.core.InvalidKeyspaceException; +import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; @@ -44,10 +51,15 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.InOrder; +import org.mockito.Mock; +import org.slf4j.LoggerFactory; public class ChannelPoolInitTest extends ChannelPoolTestBase { + @Mock private Appender appender; + @Test public void should_initialize_when_all_channels_succeed() throws Exception { when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(3); @@ -98,6 +110,142 @@ public void should_initialize_when_all_channels_fail() throws Exception { factoryHelper.verifyNoMoreCalls(); } + @Test + public void should_count_both_metrics_when_a_node_fails_on_auth_and_transport() { + // One endpoint, several addresses: ChannelFactory reports a single failure with the others + // attached as suppressed, and it promotes the authentication failure over the transport ones. + // isAuthOnly() is false for that mix, which is right for errors.connection.init -- most of the + // node really is unreachable -- but routing it there *alone* would make + // errors.connection.auth unreachable for any node whose endpoint is a name (SNI/cloud proxy, + // a client route, a translator with resolve-addresses = false), because this method is the + // driver's only writer of that counter. An operator watching it would see zero while every + // connect failed on credentials. Both happened, so both are counted. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(1); + + AuthenticationException authError = + new AuthenticationException(node.getEndPoint(), "mock auth failure"); + authError.addSuppressed(new Exception("mock connection refused")); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).failure(node, authError).build(); + + CompletionStage poolFuture = + ChannelPool.init(node, null, NodeDistance.LOCAL, context, "test"); + + factoryHelper.waitForCalls(node, 1); + + assertThatStage(poolFuture).isSuccess(); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.CONNECTION_INIT_ERRORS, null); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.AUTHENTICATION_ERRORS, null); + } + + @Test + public void should_count_only_the_auth_metric_when_every_address_fails_on_auth() { + // The other side of the same decision: nothing but authentication failed, so + // errors.connection.init must stay untouched. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(1); + + AuthenticationException authError = + new AuthenticationException(node.getEndPoint(), "mock auth failure"); + authError.addSuppressed(new AuthenticationException(node.getEndPoint(), "mock auth failure")); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).failure(node, authError).build(); + + CompletionStage poolFuture = + ChannelPool.init(node, null, NodeDistance.LOCAL, context, "test"); + + factoryHelper.waitForCalls(node, 1); + + assertThatStage(poolFuture).isSuccess(); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.AUTHENTICATION_ERRORS, null); + verify(nodeMetricUpdater, never()) + .incrementCounter(DefaultNodeMetric.CONNECTION_INIT_ERRORS, null); + } + + @Test + public void should_count_the_auth_metric_when_a_keyspace_failure_is_what_surfaced() { + // The same mix, but with a third kind of failure in it -- and ChannelFactory#surfacedFailure + // ranks an invalid keyspace above an authentication one, so *that* is what arrives here with + // the auth failure attached as suppressed. A node whose endpoint is a name expanding to two + // addresses: one rejects the credentials, the other answers but has no such keyspace. + // + // Testing the type of what arrived would leave errors.connection.auth at zero here, which is + // the very blind spot the branch above exists to close: this method is the driver's only + // writer of that counter. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(1); + + InvalidKeyspaceException surfaced = new InvalidKeyspaceException("invalid keyspace"); + surfaced.addSuppressed(new AuthenticationException(node.getEndPoint(), "mock auth failure")); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).failure(node, surfaced).build(); + + CompletionStage poolFuture = + ChannelPool.init(node, null, NodeDistance.LOCAL, context, "test"); + + factoryHelper.waitForCalls(node, 1); + + assertThatStage(poolFuture).isSuccess(); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.CONNECTION_INIT_ERRORS, null); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.AUTHENTICATION_ERRORS, null); + } + + @Test + public void should_warn_about_credentials_when_a_fatal_failure_is_what_surfaced() { + // The failure the pool has to *act* on and the failure the operator has to *hear* about are + // not always the same one, and the routing used to decide both. A node whose endpoint is a name + // expanding to two addresses: one rejects the credentials, the other rejects the protocol + // version -- which, for an identified node, ChannelFactory#surfacedFailure treats as node-wide + // and promotes. Acting on it is right: forceDown, and nothing in the driver reverses that. But + // it used to be the whole story, so the operator got a protocol-version message, a climbing + // errors.connection.auth counter, and no line at any level about the login that was refused -- + // reachable only by reading getSuppressed() off the logged throwable. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(1); + + UnsupportedProtocolVersionException surfaced = + UnsupportedProtocolVersionException.forSingleAttempt( + node.getEndPoint(), DefaultProtocolVersion.V4); + surfaced.addSuppressed(new AuthenticationException(node.getEndPoint(), "mock auth failure")); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).failure(node, surfaced).build(); + + Logger logger = (Logger) LoggerFactory.getLogger(ChannelPool.class); + Level levelBefore = logger.getLevel(); + logger.setLevel(Level.WARN); + logger.addAppender(appender); + try { + ChannelPool.init(node, null, NodeDistance.LOCAL, context, "test"); + factoryHelper.waitForCalls(node, 1); + + // Still acted on as before. + verify(eventBus, VERIFY_TIMEOUT) + .fire(TopologyEvent.forceDown(node.getBroadcastRpcAddress().get())); + verify(nodeMetricUpdater, VERIFY_TIMEOUT.times(1)) + .incrementCounter(DefaultNodeMetric.AUTHENTICATION_ERRORS, null); + + // And now also reported. + ArgumentCaptor logs = ArgumentCaptor.forClass(ILoggingEvent.class); + verify(appender, VERIFY_TIMEOUT.atLeastOnce()).doAppend(logs.capture()); + assertThat(logs.getAllValues()) + .anySatisfy( + event -> { + assertThat(event.getLevel()).isEqualTo(Level.WARN); + assertThat(event.getFormattedMessage()) + .contains("authentication failed on some of the node's addresses"); + }); + } finally { + logger.detachAppender(appender); + logger.setLevel(levelBefore); + } + } + @Test public void should_indicate_when_keyspace_failed_on_all_channels() { when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_POOL_LOCAL_SIZE)).thenReturn(3); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java new file mode 100644 index 00000000000..b9a02ddadab --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class AddressUtilsTest { + + @Test + public void should_recognize_a_hostname_whether_resolved_or_not() { + assertThat( + AddressUtils.carriesName(InetSocketAddress.createUnresolved("host.example.com", 9042))) + .isTrue(); + // Eagerly resolved by the constructor, but still a name. + assertThat(AddressUtils.carriesName(new InetSocketAddress("localhost", 9042))).isTrue(); + } + + @Test + public void should_not_mistake_an_ip_literal_for_a_hostname() throws Exception { + assertThat(AddressUtils.carriesName(new InetSocketAddress("127.0.0.1", 9042))).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("127.0.0.1", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("::1", 9042))).isFalse(); + // Built from raw bytes, so it carries no name at all and getHostString() falls back to the + // literal -- without triggering the reverse lookup that getHostName() would. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042))) + .isFalse(); + } + + @Test + public void should_report_a_bracketed_ipv6_literal_as_a_literal() { + // The spelling extract() preserves: it splits a contact point on its *last* colon, so + // "[2001:db8::5]:9042" yields the host string "[2001:db8::5]", brackets and all -- and + // InetAddress.getAllByName() accepts that form, so the configuration works end to end. + // Guava's isInetAddress() rejects brackets (only forUriString/isUriInetAddress take them), so + // testing the bare form alone would call this a host name. That is not cosmetic: it would fire + // DefaultEndPoint#equals's mixed unresolved/resolved warning and burn its once-per-JVM canary + // on a message whose stated hazards are all false for a literal, and it would send + // ChannelFactory#reattachHostname down the name-wins branch, relabelling even a + // resolver-redirected candidate. + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("[2001:db8::5]", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("[::1]", 9042))) + .isFalse(); + // Brackets and a zone together, which is what a link-local contact point looks like. + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("[fe80::1%eth0]", 9042))) + .isFalse(); + // Still a name when what is inside the brackets is not an address -- the check must not + // degenerate into "starts with a bracket". + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("[not.an.ip]", 9042))) + .isTrue(); + } + + @Test + public void should_report_a_scoped_ipv6_literal_as_a_literal() { + // Guava's isInetAddress() accepts a zone suffix, so a scoped link-local literal is correctly + // classified as a literal rather than as a name. Pinned because the opposite was once assumed: + // ChannelFactory#reattachHostname takes its IP-literal branch on the strength of this, and that + // branch has to cope with a zone it cannot parse. + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("fe80::1%eth0", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("fe80::1%1", 9042))) + .isFalse(); + // A name that merely contains a '%' is still a name. + assertThat( + AddressUtils.carriesName(InetSocketAddress.createUnresolved("od%d.example.com", 9042))) + .isTrue(); + } + + @Test + public void should_report_an_explicitly_named_address_as_a_name() throws Exception { + // A resolver may label its results with a name of its own; that is still a name. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress( + InetAddress.getByAddress("cname.example.com", new byte[] {10, 0, 0, 1}), 9042))) + .isTrue(); + } + + @Test + public void should_flip_for_a_resolved_literal_once_its_name_is_cached() throws Exception { + // The documented instability, pinned so it is not rediscovered as a surprise: a *resolved* + // address has no host string of its own. getHostString() renders InetAddress's cached hostName + // field, which getHostName() fills in with a reverse-DNS name -- and DefaultSslEngineFactory + // calls getHostName() while building an engine, on the very instance an endpoint holds. So the + // same instance answers differently before and after the first TLS handshake to that node. + InetSocketAddress resolvedLiteral = new InetSocketAddress("127.0.0.1", 9042); + assertThat(AddressUtils.carriesName(resolvedLiteral)).isFalse(); + + String reverseName = resolvedLiteral.getAddress().getHostName(); + assumeThat(reverseName) + .isNotEqualTo("127.0.0.1"); // no PTR record for loopback: nothing to show + + assertThat(AddressUtils.carriesName(resolvedLiteral)).isTrue(); + // An unresolved address has no such field, which is why callers that need a stable answer + // restrict themselves to those. + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("127.0.0.1", 9042))) + .isFalse(); + } + + @Test + public void should_strip_a_host_name_without_looking_anything_up() throws Exception { + InetSocketAddress labelled = + new InetSocketAddress( + InetAddress.getByAddress("db.example.com", new byte[] {10, 0, 0, 2}), 9042); + + InetSocketAddress stripped = AddressUtils.stripHostName(labelled); + + assertThat(stripped).isNotNull(); + assertThat(stripped.getHostString()).isEqualTo("10.0.0.2"); + assertThat(stripped.getPort()).isEqualTo(9042); + assertThat(stripped.getAddress()).isEqualTo(labelled.getAddress()); // same bytes + assertThat(AddressUtils.carriesName(stripped)).isFalse(); + } + + @Test + public void should_not_strip_an_unresolved_address() { + // Nothing to strip: there are no bytes to rebuild from. + assertThat(AddressUtils.stripHostName(InetSocketAddress.createUnresolved("host", 9042))) + .isNull(); + } + + @Test + public void should_not_invent_a_scope_for_an_unscoped_ipv6_address() throws Exception { + // Inet6AddressHolder.init treats any scope_id >= 0 as zone-present, so handing it the 0 that an + // unscoped address reports would render "...%0" -- which would reach node metric tags through + // the endpoint's toString(), and would break carriesName's resolved branch. + Inet6Address unscoped = (Inet6Address) InetAddress.getByName("2001:db8::5"); + assertThat(unscoped.getScopeId()).isZero(); + + InetAddress relabelled = AddressUtils.withHostName("db.example.com", unscoped); + + assertThat(relabelled.getHostAddress()).isEqualTo("2001:db8:0:0:0:0:0:5"); + assertThat(relabelled.getHostName()).isEqualTo("db.example.com"); + } + + @Test + public void should_keep_a_real_ipv6_scope_when_relabelling() throws Exception { + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + Inet6Address scoped = Inet6Address.getByAddress(null, linkLocal, 7); + assertThat(scoped.getScopeId()).isEqualTo(7); + + InetAddress relabelled = AddressUtils.withHostName("db.example.com", scoped); + + assertThat(((Inet6Address) relabelled).getScopeId()).isEqualTo(7); + assertThat(relabelled.getHostAddress()).endsWith("%7"); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java index 9a23f368546..e658f843360 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -186,12 +186,12 @@ private void requireSystemClientRoutesTable(CqlSession admin) { } private InetSocketAddress tryResolve(ClientRoutesTopologyMonitor handler, UUID hostId) { + // resolve() is an in-memory cache lookup that hands the route hostname over unresolved -- the + // connection layer resolves it -- so the only failure left is the monitor being closed. try { return handler.resolve(hostId); } catch (IllegalStateException e) { return null; - } catch (UnknownHostException e) { - throw new RuntimeException("DNS resolution failed for host_id=" + hostId, e); } } @@ -206,7 +206,9 @@ private NodeClassification classifyNodes(CqlSession session) { NodeClassification result = new NodeClassification(); for (Node node : session.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // getHostString() rather than getAddress().getHostAddress(): a client route is handed over + // unresolved (the connection layer resolves it), so getAddress() is null for proxied nodes. + String ip = addr.getHostString(); UUID hostId = node.getHostId(); boolean connected = node.getOpenConnections() > 0; LOG.info( @@ -319,7 +321,8 @@ private Map collectHostIds(CcmBridge ccm, int nodeCount, String t .build()) { for (Node node : adminSession.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // See classifyNodes(): a client route is unresolved, so getAddress() would be null. + String ip = addr.getHostString(); Integer nodeId = ipToNodeId.get(ip); if (nodeId != null && node.getHostId() != null) { hostIds.put(nodeId, node.getHostId()); @@ -540,7 +543,10 @@ public void should_refresh_routes_after_table_update() throws Exception { () -> { InetSocketAddress resolved = handler.resolve(hostId); assertThat(resolved).isNotNull(); - assertThat(resolved.getAddress().getHostAddress()).isEqualTo(nodeAddr); + // The route is returned unresolved on purpose -- ChannelFactory resolves it through + // Netty's AddressResolverGroup -- so assert on the host string, not getAddress(). + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo(nodeAddr); assertThat(resolved.getPort()).isEqualTo(9042); }); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java index 26658bd76d1..dcede822fca 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java @@ -235,6 +235,12 @@ private CqlSession newSession(ProgrammaticDriverConfigLoaderBuilder loaderBuilde .withDuration(DefaultDriverOption.HEARTBEAT_TIMEOUT, Duration.ofMillis(500)) .withDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT, Duration.ofSeconds(2)) .withDuration(DefaultDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(1)) + // These tests exercise heartbeat behavior only. The contact-point reconnection + // fallback appends the contact points to a reconnection plan that has run out of live + // nodes, so a round tries roughly twice as many addresses -- and every one of those + // attempts sends an OPTIONS as part of protocol init. countHeartbeats() filters on + // isOptionRequest() alone, so it counts those alongside real heartbeats. Disable it. + .withBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false) .build(); return SessionUtils.newSession(SIMULACRON_RULE, loader); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index 4e9eefebf63..e9d9d715d19 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -25,9 +25,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; @@ -37,16 +39,21 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.config.typesafe.DefaultProgrammaticDriverConfigLoaderBuilder; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -60,6 +67,47 @@ public class MockResolverIT { private static final int CLUSTER_WAIT_SECONDS = 20; // Maximal wait time for cluster nodes to get up + /** + * A loopback address no node is ever started on, outside the {@code 127.0.1.} prefix CCM hands + * its nodes: a connection attempt that dials it is refused immediately. + */ + private static final String DEAD_ADDRESS = "127.0.0.11"; + + private static final ch.qos.logback.classic.Logger CHANNEL_FACTORY_LOGGER = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(ChannelFactory.class); + + private ListAppender channelFactoryAppender; + private Level originalChannelFactoryLevel; + + @Before + public void startCapturingChannelFactoryLogs() { + // ChannelFactory reports a candidate it gave up on at DEBUG, which is the only externally + // visible evidence that the multi-address fallback did any work. + originalChannelFactoryLevel = CHANNEL_FACTORY_LOGGER.getLevel(); + CHANNEL_FACTORY_LOGGER.setLevel(Level.DEBUG); + channelFactoryAppender = new ListAppender<>(); + // ChannelFactory logs from every I/O thread of the session under test, and the assertions read + // the list while those threads are still winding down; ListAppender's own ArrayList is not safe + // for that (appends are unsynchronized, and iterating one would throw). + channelFactoryAppender.list = new CopyOnWriteArrayList<>(); + channelFactoryAppender.start(); + CHANNEL_FACTORY_LOGGER.addAppender(channelFactoryAppender); + } + + @After + public void stopCapturingChannelFactoryLogs() { + CHANNEL_FACTORY_LOGGER.detachAppender(channelFactoryAppender); + channelFactoryAppender.stop(); + CHANNEL_FACTORY_LOGGER.setLevel(originalChannelFactoryLevel); + } + + /** The formatted messages {@code ChannelFactory} logged during the current test. */ + private List channelFactoryLogMessages() { + return channelFactoryAppender.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .collect(Collectors.toList()); + } + private static void waitForAllNodesUp(CqlSession session, int expectedNodes) { Awaitility.await() .atMost(CLUSTER_WAIT_SECONDS, TimeUnit.SECONDS) @@ -84,7 +132,6 @@ public void should_connect_with_mocked_hostname() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -101,16 +148,95 @@ public void should_connect_with_mocked_hostname() { for (Node node : nodes) { LOG.trace("Found metadata node: {}", node); } - Set filteredNodes; - filteredNodes = + // Selected by the address the connection reached, not by the contact-point name: the node + // is + // identified by its own address (see below), so the name no longer appears in toString(). + String nodeIp = ccmBridge.getNodeIpAddress(1); + Set filteredNodes = nodes.stream() - .filter(x -> x.toString().contains("test.cluster.fake")) + .filter( + x -> { + InetSocketAddress resolved = (InetSocketAddress) x.getEndPoint().resolve(); + return !resolved.isUnresolved() + && nodeIp.equals(resolved.getAddress().getHostAddress()); + }) .collect(Collectors.toSet()); assertThat(filteredNodes).hasSize(1); - InetSocketAddress address = - (InetSocketAddress) filteredNodes.iterator().next().getEndPoint().resolve(); - assertTrue(address.isUnresolved()); + Node node = filteredNodes.iterator().next(); + InetSocketAddress address = (InetSocketAddress) node.getEndPoint().resolve(); + // ChannelFactory pins the control connection's endpoint to the address it actually reached, + // and DefaultTopologyMonitor#buildNodeEndPoint keeps that address for the control node, so + // resolution yields that concrete IP rather than the hostname. + assertFalse(address.isUnresolved()); + assertThat(address.getAddress().getHostAddress()).isEqualTo(nodeIp); + // ... while still carrying the configured name as its label, so the TLS peer host and the + // Kerberos service name are what the operator wrote, with no reverse lookup. + assertThat(address.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(address.getAddress().getHostName()).isEqualTo("test.cluster.fake"); + // The node's *identity*, though, is its own address rather than the contact point's. The + // reconnection fallback re-offers the contact points every round, so a name-derived prefix + // would be acquired by each successive control node in turn, and two live nodes sharing one + // prefix means the older one's clearMetrics() deletes the newcomer's series. + assertThat(node.getEndPoint().asMetricPrefix()) + .isEqualTo(nodeIp.replace('.', '_') + ":9042"); + } + } + } + + @Test + public void should_connect_when_first_dns_entry_is_non_responsive() { + final int numberOfNodes = 2; + DriverConfigLoader loader = + new DefaultProgrammaticDriverConfigLoaderBuilder() + .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) + .withStringList( + TypedDriverOption.CONTACT_POINTS.getRawOption(), + Collections.singletonList("test.cluster.fake:9042")) + .build(); + + CqlSessionBuilder builder = new CqlSessionBuilder().withConfigLoader(loader); + try (CcmBridge ccmBridge = + CcmBridge.builder().withNodes(numberOfNodes).withIpPrefix("127.0.1.").build()) { + MultimapHostResolverProvider.removeResolverEntries("test.cluster.fake"); + // Nothing is ever started on DEAD_ADDRESS, so it is the dead record. + MultimapHostResolverProvider.addResolverEntry("test.cluster.fake", DEAD_ADDRESS); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(1)); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(2)); + ccmBridge.create(); + ccmBridge.start(); + + // ChannelFactory shuffles a name's expanded addresses per connection attempt, so whether the + // dead record is dialed first is a coin toss (about 1 in 3 here). Every session must connect + // either way; the loop hunts for an iteration that demonstrably dialed the dead record first + // and fell through to a live one, which is the behavior this test exists to pin down. + // Twenty misses in a row would have probability (2/3)^20, about 0.03%. + boolean sawFallback = false; + for (int attempt = 0; attempt < 20 && !sawFallback; attempt++) { + try (CqlSession session = builder.build()) { + waitForAllNodesUp(session, numberOfNodes); + ResultSet rs = session.execute("select * from system.local where key='local'"); + assertThat(rs).isNotNull(); + List rows = rs.all(); + assertThat(rows).hasSize(1); + Collection nodes = session.getMetadata().getNodes().values(); + assertThat(nodes).hasSize(numberOfNodes); + } + sawFallback = + channelFactoryLogMessages().stream() + .anyMatch( + message -> + message.contains(DEAD_ADDRESS) && message.contains("trying next address")); } + + // The connections above only survived a dead-first ordering because the candidate loop moved + // past the dead record. + assertThat(sawFallback) + .as( + "expected at least one connection attempt to dial %s first and fall through", + DEAD_ADDRESS) + .isTrue(); } } @@ -119,7 +245,6 @@ public void replace_cluster_test() { final int numberOfNodes = 3; DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -152,12 +277,14 @@ public void replace_cluster_test() { while (iterator.hasNext()) { LOG.trace("Metadata node: " + iterator.next().toString()); } - Set filteredNodes; - filteredNodes = - nodes.stream() - .filter(x -> x.toString().contains("test.cluster.fake")) - .collect(Collectors.toSet()); - assertThat(filteredNodes).hasSize(1); + // Exactly one node -- the one the control connection came up on -- still names the contact + // point. It is the endpoint's *label* rather than its identity: the node is identified by the + // address the connection reached (see should_connect_with_mocked_hostname), while the label + // is + // what TLS and Kerberos read. Re-resolution itself no longer depends on any node holding the + // name: MetadataManager retains the contact points, and the control-connection reconnection + // fallback re-offers them. + assertThat(nodesNamingTheContactPoint(nodes)).hasSize(1); } try (CcmBridge ccmBridge = CcmBridge.builder().withNodes(numberOfNodes).withIpPrefix("127.0.1.").build()) { @@ -175,23 +302,37 @@ public void replace_cluster_test() { while (iterator.hasNext()) { LOG.trace("Metadata node: " + iterator.next().toString()); } - Set filteredNodes; - filteredNodes = - nodes.stream() - .filter(x -> x.toString().contains("test.cluster.fake")) - .collect(Collectors.toSet()); - if (filteredNodes.size() == 0) { + Set filteredNodes = nodesNamingTheContactPoint(nodes); + if (filteredNodes.isEmpty()) { LOG.error( - "No metadata node with \"test.cluster.fake\" substring. The unresolved endpoint socket was likely " - + "replaced with resolved one."); + "No metadata node whose endpoint is labelled \"test.cluster.fake\". The label was likely " + + "dropped when the endpoint was rebuilt."); } else if (filteredNodes.size() > 1) { fail( - "Somehow there is more than 1 node in metadata with unresolved hostname. This should not ever happen."); + "Somehow there is more than 1 node in metadata labelled with the contact point. This should not ever happen."); } } session.close(); } + /** + * The nodes whose endpoint carries the contact-point name as its host string. + * + *

Deliberately not {@code toString().contains(...)}: a node is identified by the address the + * connection reached, so the name lives on as the endpoint's label -- what the SSL engine + * and the Kerberos service name read -- rather than in its identity. + */ + private static Set nodesNamingTheContactPoint(Collection nodes) { + return nodes.stream() + .filter( + node -> { + SocketAddress resolved = node.getEndPoint().resolve(); + return resolved instanceof InetSocketAddress + && "test.cluster.fake".equals(((InetSocketAddress) resolved).getHostString()); + }) + .collect(Collectors.toSet()); + } + @SuppressWarnings("unused") public void run_replace_test_20_times() { for (int i = 1; i <= 20; i++) { @@ -206,7 +347,6 @@ public void run_replace_test_20_times() { public void cannot_reconnect_with_resolved_socket() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -239,10 +379,7 @@ public void cannot_reconnect_with_resolved_socket() { while (iterator.hasNext()) { LOG.trace("Metadata node: " + iterator.next().toString()); } - filteredNodes = - nodes.stream() - .filter(x -> x.toString().contains("test.cluster.fake")) - .collect(Collectors.toSet()); + filteredNodes = nodesNamingTheContactPoint(nodes); assertThat(filteredNodes).hasSize(1); } int counter = 0; @@ -272,10 +409,7 @@ public void cannot_reconnect_with_resolved_socket() { while (iterator.hasNext()) { LOG.trace("Metadata node: " + iterator.next().toString()); } - filteredNodes = - nodes.stream() - .filter(x -> x.toString().contains("test.cluster.fake")) - .collect(Collectors.toSet()); + filteredNodes = nodesNamingTheContactPoint(nodes); if (filteredNodes.size() > 1) { fail( "Somehow there is more than 1 node in metadata with unresolved hostname. This should not ever happen."); diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index ae44feea3ea..87c542880c8 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -30,14 +30,78 @@ topologies, an address translation component can be plugged in. ----- -Each node in the Cassandra cluster is uniquely identified by an IP address that the driver will use -to establish connections. +Each node in the Cassandra cluster is uniquely identified by an address that the driver will use to +establish connections. * for contact points, these are provided as part of configuring the `CqlSession` object; * for other nodes, addresses will be discovered dynamically, either by inspecting `system.peers` on already connected nodes, or via push notifications received on the control connection when new nodes are discovered by gossip. +That address does not have to be an IP: see [multi-address resolution](#multi-address-resolution) +below. + + +### Multi-address resolution + +An address the driver holds as a **hostname** is resolved at connection time, and expanded to *every* +address the name maps to; each one is tried in turn until a connection succeeds. So a single +unreachable IP behind a multi-record name no longer fails the connection, and a DNS record change is +picked up on the next connection attempt rather than requiring a restart. + +This applies wherever the driver holds a name rather than an IP: + +* **contact points** — kept as you wrote them, and re-resolved per connection attempt. (The + `advanced.resolve-contact-points` option, which used to resolve them once at startup and turn each + A-record into a separate `Node`, is deprecated and has no effect.) The exception is a contact point + you pass programmatically as an *already-resolved* `InetSocketAddress`: there is no name left in it + to re-resolve, so it stays bound to that one address. Use + `InetSocketAddress.createUnresolved(host, port)`, or configure the contact points as strings under + `basic.contact-points`, if you want the name expanded — see `SessionBuilder.addContactPoints`. +* the **Cloud SNI proxy** address, and a **client route** hostname (see below); +* whatever a custom [`AddressTranslator`](#driver-side-address-translation) hands back — + `SubnetAddressTranslator` returns a name by default, under `resolve-addresses = false`. + +Two configuration options matter: + +* `advanced.connection.max-candidate-addresses` (default `5`) caps how many addresses **one** + connection attempt tries. Each one costs a TCP connect plus the protocol handshake, and the + attempts are serial, so this bounds what a single attempt can cost in time. Set it to `1` to + restore the pre-multi-address behaviour of one address per attempt. +* `advanced.control-connection.reconnection.fallback-to-original-contact-points` (default `true`) + appends the original contact points to the control connection's reconnection plan, after the live + nodes the load balancing policy offers. This is the driver's DNS re-resolution path: a node + discovered from `system.peers` holds an already-resolved address that is never re-resolved, and so + does the node the control connection is on — it is registered under the address it reached, not + under the contact point it was reached through — so falling back to the contact-point hostnames is + what lets a reconnect pick up new IPs. Turning it off therefore disables DNS re-resolution + altogether, rather than narrowing it. + + Two exceptions, where the option is close to a no-op. Under an `AddressTranslator` that returns a + hostname, every node's endpoint stays unresolved and re-expands per attempt regardless. And in a + Cloud (SNI) session every endpoint is an `SniEndPoint`, which does the same — and there the append + is skipped altogether unless the live-node plan came back empty, because the topology monitor + re-resolves node addresses itself. Turning the option off in those deployments removes only that + empty-plan fallback. + + A client-routes session is not a third exception. Its endpoints re-expand a route hostname per + attempt only while that node has a route; a node without one falls back to a static, + already-resolved address that is never re-resolved. The driver therefore reports such a session as + re-resolving its own addresses only while *every* known node has a live route. In a partially + routed cluster the contact points are appended as usual, and this option is the only DNS + re-resolution the route-less nodes get. + +Resolution goes through Netty's configured `AddressResolverGroup`, so a resolver installed via +`NettyOptions.afterBootstrapInitialized()` is honoured — including `DnsAddressResolverGroup` for +non-blocking lookups. With Netty's default resolver the lookup blocks the Netty I/O event loop it +runs on (never the admin loop), which is the same behaviour an unresolved address had before, so the +JVM DNS cache settings (`networkaddress.cache.ttl`) matter more than they used to. + +Custom `EndPoint` implementations take part in this: return an +[unresolved](https://docs.oracle.com/javase/8/docs/api/java/net/InetSocketAddress.html#isUnresolved--) +`InetSocketAddress` from `resolve()` and the driver expands it. Implementations must **not** resolve +names themselves and must not block — see the `EndPoint.resolve()` javadoc. + ### Cassandra-side configuration @@ -183,13 +247,19 @@ datastax-java-driver { #### DNS resolution -DNS is resolved at connection time (not at route discovery time). The driver delegates to -`InetAddress.getByName()`, which is a blocking call that uses the JVM's built-in DNS cache -(30 s default TTL in the JDK). Because this runs on Netty I/O threads, slow or unresponsive -DNS can block connection establishment and impact driver throughput. To mitigate this, configure -the JVM DNS cache TTL via the `networkaddress.cache.ttl` security property (e.g. in -`$JAVA_HOME/conf/security/java.security` or programmatically with -`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`). +DNS is resolved at connection time (not at route discovery time), and through the same mechanism as +every other address the driver connects to: the route's hostname is handed to the connection layer +unresolved, and Netty's configured `AddressResolverGroup` expands it. A custom resolver installed via +`NettyOptions.afterBootstrapInitialized()` therefore applies to client routes as well, and a hostname +that maps to several addresses has all of them tried in turn. + +With Netty's default (JDK) resolver the lookup is a blocking `InetAddress` call that uses the JVM's +built-in DNS cache (30 s default TTL in the JDK). It runs on a Netty I/O event loop — never on the +admin event loop that drives control-connection reconnects — so it delays the connection attempt +itself. It is therefore worth configuring the JVM DNS cache TTL via the `networkaddress.cache.ttl` +security property (e.g. in `$JAVA_HOME/conf/security/java.security` or programmatically with +`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`), or installing +`DnsAddressResolverGroup` for non-blocking resolution. - **Route-map refresh** — the driver re-queries `system.client_routes` and atomically swaps the in-memory route map in two situations: @@ -218,6 +288,12 @@ datastax-java-driver.advanced.address-translator.class = FixedHostNameAddressTra advertised-hostname = proxyhostname ``` +The advertised hostname is handed on unresolved, so it is expanded on every connection attempt like +any other name (see [Multi-address resolution](#multi-address-resolution)): if the proxy is fronted by several A-records, +all of them are tried, and a change to the records is picked up without restarting the session. The +addresses are tried in the order the resolver returned them rather than shuffled, because a fixed +proxy hostname says nothing about whether its addresses lead to the same place. + ### Fixed proxy hostname per subnet When running Cassandra in a private network and accessing it from outside of that private network via some kind of diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 83963f94775..cba0275cf65 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -61,6 +61,309 @@ Binary compatibility is unaffected — an already-compiled subclass keeps workin override either method, you have to widen your override to `public` in order to recompile: Java does not allow an override to reduce visibility. +#### Contact points are expanded to all their DNS addresses at connection time + +Contact points backed by a hostname are now kept unresolved and expanded to **all** the IP +addresses the hostname maps to, at connection time. Previously only the first address returned by +DNS was tried, so a single non-responsive IP behind a multi-record hostname could fail the initial +connection (or a control-connection reconnect) even when the other addresses were healthy. No +configuration change is required to benefit from this. + +The expansion goes through Netty's configured `AddressResolverGroup`, which is the resolver an +unresolved address already reached when it was handed to `Bootstrap.connect()`. A custom resolver +installed through `NettyOptions.afterBootstrapInitialized()` therefore keeps working; note that the +driver now calls `resolveAll()` on it, where previously `Bootstrap.connect()` called `resolve()`. +Every resolver built on Netty's own base classes implements both. As before, with Netty's default +(JDK) resolver the lookup blocks the I/O event loop it runs on — install `DnsAddressResolverGroup` +for non-blocking resolution. + +`Bootstrap.disableResolver()` is still honored, in the sense that the driver resolves nothing and +passes the address through as-is. That now requires the address to be usable as-is, which is a +narrower contract than before: contact-point hostnames are always kept unresolved, and the Cloud/SNI +proxy address and client-route hostnames no longer resolve themselves either, so with resolution +disabled none of them can connect. The driver fails such an attempt with a message naming the +address and the reason, rather than the bare `UnresolvedAddressException` Netty would otherwise +raise. If you disable the resolver, supply already-resolved addresses. + +The same now applies to the two other address sources that used to perform their own JVM DNS lookups: +the Cloud/SNI proxy address and client-route hostnames are expanded by the connection layer too. A +custom Netty resolver applies to them for the first time, and a proxy hostname with several A-records +has all of them tried within a single connection attempt. + +There is **no public API change**: `EndPoint.resolve()` keeps its signature and is not deprecated. +Third-party `EndPoint` implementations keep working unchanged, with one new expectation — an +implementation should return its address as-is rather than looking names up itself, since resolution +now happens in the connection layer and `resolve()` is called from an event loop. + +**If you call `node.getEndPoint().resolve()` yourself, check how you read the host.** Because +resolution moved to the connection layer, that address is no longer always resolved: + +| Deployment | `resolve()` returns | +| --- | --- | +| Nodes discovered from `system.peers`, and the node the control connection is on | resolved, as before | +| Nodes whose address came from an `AddressTranslator` that returns a name | that name, **unresolved** | +| Cloud / SNI proxy | the configured proxy hostname, **unresolved** | +| Cloud private-endpoint client routes (nodes that have a route) | the route hostname, **unresolved** | + +The second row catches more deployments than it looks: `SubnetAddressTranslator` returns names under +its default `advanced.address-translator.resolve-addresses = false`, so peers reached through it are +unresolved even though the first row says otherwise for peers in general. + +For every row marked **unresolved** — the last three — `InetSocketAddress.getAddress()` now returns +`null`, so a call such as +`((InetSocketAddress) node.getEndPoint().resolve()).getAddress().getHostAddress()` throws a +`NullPointerException` where it previously worked. Use `getHostString()` instead: it returns +whichever of a hostname or an IP literal the address carries, for both the resolved and the +unresolved case, and never triggers a reverse lookup. + +Only part of the second row is new. `SubnetAddressTranslator` already returned an unresolved address +before this release, so code that survived it is unaffected; `FixedHostNameAddressTranslator` and +`Ec2MultiRegionAddressTranslator` used to return resolved addresses and no longer do, so that call +worked on every prior release under those two and now throws. + +As part of this change: + +- `advanced.resolve-contact-points` is deprecated and now has **no effect**; setting it to `true` + logs a warning at startup. Contact points are always kept as unresolved hostnames and expanded at + connection time. An already-resolved `InetSocketAddress` passed programmatically is still used as + provided, with no further expansion. + + If you had set it to `true`, note what that setting used to buy you, since the driver no longer + offers it: each address a contact-point hostname resolved to became its own `Node`, so each had + its own entry in `Metadata.getNodes()` and its own node-level metrics, and the load balancing + policy's query plan advanced address by address. A hostname is now a single `Node`, and its + addresses are tried inside one connection attempt. Resolution + also used to happen once, at startup; it now happens per connection attempt, through Netty's + configured resolver, which is what lets the driver pick up changed DNS records — but it means the + resolver is consulted far more often than before, so the JVM DNS cache settings + (`networkaddress.cache.ttl`) matter more than they used to. +- **The node the control connection is on is identified by the address it is connected to, not by + the contact point it was reached through.** This shows up in node-level metrics: with a + hostname contact point, the node the control connection came up on used to report under a prefix + derived from that hostname (`nodes.cluster_example_com:9042.*`, and the `node` tag likewise) — + which was never really its own name, since any node can end up being the one a contact point + reaches. It now reports under its own address, like every other node + (`nodes.10_0_0_2:9042.*`). If you scraped a node series named after your contact-point hostname, + expect it to move once, to that node's address. Nothing to configure. With + `TaggingMetricIdGenerator` the `node` tag moves the same way, from `cluster.example.com:9042` to + `/10.0.0.2:9042` — the form the driver already uses for nodes discovered from `system.peers`. + What does **not** change is what the node connects to: its endpoint still resolves to that + address carrying the name you configured, so TLS hostname verification still sees + `cluster.example.com` without a reverse-DNS lookup. The Kerberos service name is the exception, + and behaves exactly as it did before this change rather than better: `DseGssApiAuthProvider` + builds the service principal from `InetAddress.getCanonicalHostName()`, which consults the name + service and ignores the label the address carries. +- `advanced.control-connection.reconnection.fallback-to-original-contact-points` now defaults to + `true` (previously `false`). This is also the driver's DNS re-resolution path: a metadata node + normally holds an address decoded from the peers table, which is already resolved and so never + picks up a DNS change, and falling back to the original contact points on control-connection + reconnect is what re-reads the records once the live-node plan is exhausted. (The exception is an + `AddressTranslator` that returns a hostname — `SubnetAddressTranslator` does, unless you set + `advanced.address-translator.resolve-addresses = true`. Such an endpoint is re-expanded on every + connection attempt like any other name, so its node is not pinned to one address in the first + place. Its addresses are tried in the resolver's own order rather than shuffled — see below — so + its channels stay on one address in practice, but if that name maps to more than one host the + driver has no way to tell.) Setting it to `false` does **not** restore the previous behavior — it + leaves the session with no DNS re-resolution at all, which is less than either 4.19 or the + default here. Before this release the control connection's own node kept the contact point's + unresolved hostname, so every reconnect and every one of its pool connections re-read DNS with no + option involved; that node is now registered under the address it actually reached (see the + metric-prefix note above), so the contact-point fallback is the only path left. Set it to `false` + only when your contact points are IP literals, or when their records never change. (Two + deployments are exempt from all of this, because their endpoints are unresolved to begin with and + re-expand per attempt whatever the option says: an `AddressTranslator` that returns a hostname, + and a Cloud (SNI) session, whose `SniEndPoint`s keep the proxy name. For Cloud the append is + skipped unless the live-node plan is empty, so neither the benefit nor the cost below applies. A + **client-routes session is not exempt**: its endpoints re-expand only for nodes that currently + have a route, and a route-less node falls back to a static, already-resolved address, so the + driver treats such a session as self-re-resolving only while every known node has a route. Below + that, the contact points are appended as usual and this option is the only DNS re-resolution the + route-less nodes get — so `false` costs them more there than anywhere else.) Note the cost + of leaving it on: the contact points are appended without being compared against the live-node + plan (at plan time + they are still hostnames, while the live nodes are already-resolved IPs), so when DNS has not + changed a reconnection round that exhausts the live nodes retries them a second time. How much + that adds depends on how many addresses each contact point resolves to — a live node is one + address, a contact point is expanded to up to `advanced.connection.max-candidate-addresses` of + them — so three contact points can append up to 15 serial attempts to a round, not 3, with the + control connection down for the whole of it. Each of those attempts also carries an identity + read, because an appended contact point is an unidentified node: one `SELECT * FROM system.local` + per candidate address, bounded by `advanced.control-connection.timeout`, which additionally + discards the column projection learned from the previous control node. +- **`FixedHostNameAddressTranslator` now returns the advertised hostname unresolved.** It used to + build `new InetSocketAddress(advertisedHostname, port)`, which resolves eagerly, and a resolved + address is passed straight through the connection layer — so every node in the cluster was pinned + to whichever single IP the JVM happened to return first, and no other address of the proxy was + ever tried. The name is now expanded per connection attempt like any other, so a proxy hostname + with several A-records fails over between them. The addresses are tried in the resolver's own + order rather than shuffled, since the translator makes no claim that they are interchangeable. + Two consequences worth knowing: `node.getEndPoint().resolve()` for these nodes now returns an + unresolved address (see the table above), and the DNS records are re-read on every connection + attempt instead of once at startup, so `networkaddress.cache.ttl` applies. A third, if you also + read schema metadata: with the peers unresolved and the control node's endpoint resolved, + `Metadata.findNode(EndPoint)` compares the two forms and resolves the unresolved side inline, so + a schema refresh costs one lookup per node. It logs a one-time warning naming + [#1006](https://github.com/scylladb/java-driver/issues/1006). One node is exempt from the + fail-over above: the node the control connection is on is registered under the address that + connection reached (see the metric-prefix note), so its own pool stays on that address until the + control connection moves and the node is re-derived from `system.peers`. Recovering an address + change for it is what the contact-point reconnection fallback is for. + + And a fourth, if you use the **tagging** metric id generator: `TaggingMetricIdGenerator` tags node + metrics with `node.getEndPoint().toString()`, which for these nodes moves from + `advertised.host/10.0.0.1:9042` to `advertised.host:9042` — and to + `advertised.host/:9042` on JDK 14 and later, since + [JDK-8225499](https://bugs.openjdk.org/browse/JDK-8225499) changed how an unresolved + `InetSocketAddress` renders. So every Micrometer or MicroProfile node series is re-tagged on + upgrade, and the tag differs between a JDK 11 and a JDK 17 runtime of the same driver version. + The default `DefaultMetricIdGenerator` is not affected: `asMetricPrefix()` reduces the endpoint to + host string plus port, which is the same either way. +- **`Ec2MultiRegionAddressTranslator` now returns the domain name unresolved**, for the same reason + and with the same consequences. The reverse (PTR) lookup that finds the name is unchanged, and the + translator still checks that the name resolves before handing it over — a PTR record whose target + has no A-record is answered with the node's original address, exactly as before. What moves into + the connection attempt is the choice of address, so a domain name with several A-records now fails + over between them instead of being pinned to the first. +- **DSE Insights reports contact points by name, not by resolved address.** With contact points kept + unresolved, the startup event's `contact_points` map goes from `{"db.example.com": + ["10.0.0.1:9042", "10.0.0.2:9042"]}` to `{"db.example.com": ["db.example.com:9042"]}`, and the + address the session actually connected to is still reported under `initial_control_connection`. + The driver does not resolve the names to fill the map back in: that would mean a blocking DNS + lookup per contact point on the admin executor, which is the thread this release worked to keep + name lookups off. + + For the same reason the map is now keyed on the host **string** rather than the host **name**. + Those differ for a contact point supplied as a resolved `InetSocketAddress` carrying no label — + which includes `new InetSocketAddress("10.0.0.1", 9042)`, since an IP literal sets no host name. + Such a contact point used to be reported under its reverse-DNS (PTR) name and is now reported + under the literal, so a `127.0.0.1` contact point that used to appear as `localhost` appears as + `127.0.0.1`. That also removes a reverse lookup per such contact point from the admin executor, + repeated at every `advanced.monitor-reporting` interval. +- **A contact point that no address can identify is now marked down before the connection attempt + completes.** Establishing which node answered — reading `host_id` from `system.local` — happens + while the driver still holds the hostname's other addresses, so a candidate address that cannot + identify itself is rejected like any other per-address failure and the next address of the same + hostname is tried. When every address has been tried and none worked, the driver fires the same + `controlConnectionFailed` event and node-down state change that a refused TCP connection + produces, where previously it moved on to the next contact point without either. Nothing to + change on your side; expect the DOWN event and its log line in a case that used to be silent. +- **One failed connection attempt can now increment two error metrics.** A node whose endpoint is a + name — an SNI/cloud proxy, a client route, or an `AddressTranslator` with + `resolve-addresses = false` — reports a single failure for the whole attempt, with the other + addresses' failures attached as suppressed exceptions. When that attempt mixes an authentication + failure with transport failures, both `errors.connection.init` and `errors.connection.auth` are + incremented for it: counting only the promoted failure would have left `errors.connection.init` at + zero while most of the node was unreachable, and counting only the transport ones would have made + `errors.connection.auth` unobservable for these deployments, since connection setup is its only + writer. If you sum the two counters to get "failed attempts", or divide one by the other, expect + the total to exceed the number of attempts on exactly those deployments. +- **A Cloud proxy given as an IP literal is no longer validated against its reverse-DNS name.** + `advanced.ssl-engine-factory.allow-dns-reverse-lookup-san` used to reach a PTR lookup for a proxy + configured as `withCloudProxyAddress(new InetSocketAddress("203.0.113.10", 9042))`, because the + addresses the driver resolved carried no hostname label. The connection layer now labels every + candidate before the SSL engine sees it, so the certificate is checked against the literal and + needs an `iPAddress` SAN; a proxy certificate carrying only a DNS SAN for the reverse name will + fail the handshake with "No subject alternative names matching IP address … found". Configure the + proxy by hostname (what secure connect bundles do) to keep DNS-SAN validation. Proxies configured + by hostname are unaffected — those lookups already carried the queried name, so the option made no + difference there either. +- **`AllNodesFailedException` groups a hostname's failures.** A contact point that resolves to + several addresses contributes one entry per contact point rather than per address, with each + address's failure attached to it as a suppressed exception. If you inspect + `AllNodesFailedException.getAllErrors()`, expect fewer entries carrying more detail; the underlying + causes are all still reachable through `Throwable.getSuppressed()`. +- **A name's addresses are tried in random order, and at most + `advanced.connection.max-candidate-addresses` of them per connection attempt** (new option, + default 5). The shuffle spreads load across a multi-record name's addresses and varies the + starting address between attempts. It applies to every contact point, and to an already-identified + node when its addresses are interchangeable — an SNI proxy or a cloud private-endpoint route, + where the proxy routes by server name so all of its addresses lead to the same node. For those the + driver spread connections across the records before this version too, inside + `SniEndPoint.resolve()`. The order is kept only for a name that may denote *different* hosts, + which is what an `AddressTranslator` can return (`SubnetAddressTranslator` does by default): one + node's connection pool converges on one address there, as it did before, and the remaining + addresses serve as fallback. The cap applies either way + and bounds what one attempt can cost, since every address + tried is a full TCP connect plus handshake — and, with wrong credentials, a rejected login + (authentication failures do not stop the walk: with a multi-record name they may be specific to + the address, e.g. a stale record pointing at a foreign cluster). Where the order is shuffled, + addresses beyond the cap are + not lost — every attempt samples a fresh random subset, so reconnection rounds reach all of them + over time; where it is kept, the cap is a hard limit and records past it are not reached, which is + still more than the single address such a name got before. Set the option to `1` to restore + one-address-per-attempt behavior. +- **The control connection can now decline a node after a successful handshake.** Once the driver + knows which node answered, it re-checks that node against what the load balancing policy and + topology events say about it, and closes the channel if it is IGNORED, forced down or removed, + moving on to the next entry in the plan. Previously a contact point reached this way was an + ephemeral object that no event had ever named, so the check found nothing and the connection was + kept. This matters most with the contact-point fallback above now on by default: if your contact + points resolve to nodes the policy excludes, expect the control connection to keep looking rather + than settle on one of them. The reason is recorded in the resulting `AllNodesFailedException`. + Worth knowing where that leaves you if the *only* reachable nodes are excluded ones — a multi-DC + deployment whose local DC is entirely down, where `DefaultLoadBalancingPolicy` has marked every + remote node IGNORED: the control connection stays down instead of parking on a remote node, so + topology and schema stop being refreshed for the duration. Recovery does not depend on it — the + connection pools keep retrying the local nodes themselves, and a successful pool connect is what + brings them back UP — but you will see the control connection cycling through its plan in the logs + for as long as the local DC is unreachable. A node that is merely `DOWN` is not affected; only + IGNORED, forced down and removed are. +- If you use `TaggingMetricIdGenerator` **and** build the Cloud proxy address yourself with + `new InetSocketAddress("proxy-host", port)` rather than through a secure connect bundle: the + `node` tag for those nodes changes once. The tag is the endpoint's `toString()`, which for an SNI + endpoint is the proxy address followed by the node's host id, and the proxy half of it is what + moves: + + | | Java 13 and earlier | Java 14 and later | + | --- | --- | --- | + | before | `proxy-host/1.2.3.4:9042:` | `proxy-host/1.2.3.4:9042:` | + | after | `proxy-host:9042:` | `proxy-host/:9042:` | + + The two "after" forms differ because `InetSocketAddress.toString()` began marking unresolved + addresses with `/` in Java 14, and the driver now keeps the configured proxy address + unresolved so it can be re-expanded to every proxy A-record. The tag no longer depends on which + proxy IP a connection happened to land on — that stability is the point, but expect a one-time + series rename in dashboards, and note that the new name depends on the JDK you run. The same + rename applies if you passed the proxy as an IP literal (`1.2.3.4/1.2.3.4:9042:` becomes + `1.2.3.4:9042:`, or `1.2.3.4/:9042:` on Java 14 and later), which is + stored unresolved too so that the endpoint's identity cannot change underneath it when something + asks the address for its reverse-DNS name. `asMetricPrefix()`, and therefore the default + `MetricIdGenerator`, keeps reporting the proxy address as you supplied it — with one exception. If + you built that address from an `InetAddress` rather than from a host string + (`new InetSocketAddress(InetAddress.getByName("1.2.3.4"), port)`), the prefix used to be computed + live and `resolve()` called `getHostName()` on that shared `InetAddress`, so from the first + connection onwards the prefix reported whatever reverse-DNS name came back. It now stays the + literal. That is the stable answer, but if a PTR record existed it is also a one-time rename. +- For advanced deployments that provide a custom `NettyOptions`: the + `afterBootstrapInitialized()` hook now runs once per logical connection to a node (previously + once per attempt, including protocol-version downgrade retries), and it receives the bootstrap + *before* the driver installs its channel handler — a handler set by the hook is replaced, and + the driver logs a one-time warning if it detects one. An `EventLoopGroup` set by the hook is + likewise ignored, and likewise warned about once: every connection attempt is now bound to an + event loop the driver picks from `ioEventLoopGroup()`, so override that method instead if you + need driver I/O on your own threads. Use the hook for channel options, attributes and + `Bootstrap.resolver(...)`; use `afterChannelInitialized()` for pipeline customization. +- Two `protected` methods that no longer have anything to do are removed. Both are in internal + packages, but they were reachable from a subclass, so a custom subclass overriding either of them + will no longer compile: + - `OptionalLocalDcHelper.checkLocalDatacenterCompatibility(String, Set)` — it warned when a + contact point's datacenter differed from the configured local DC, but contact-point nodes never + get a datacenter assigned, so it compared against `null` and warned unconditionally instead of + on a real mismatch. Note what goes with it: the separate "configured local DC matches no node" + warning is retained, but it is not the same diagnostic. If you set `local-datacenter = dc1`, + point every contact point at `dc2`, and the cluster does have live `dc1` nodes, that check finds + `dc1` among them and says nothing — so that particular misconfiguration is now silent rather + than reported in a different place. The deleted check could not report it either (it had no + datacenter to compare), which is why it goes, but the gap is real and worth naming. + - `ClientRoutesTopologyMonitor.resolveAddress(String)` — a test seam for the JVM DNS lookup that + the monitor no longer performs, now that route hostnames are resolved by the connection layer. +- For the same reason, `ClientRoutesTopologyMonitor.resolve(UUID)` — which is `public` — no longer + declares `UnknownHostException`: it hands back the route hostname unresolved instead of looking it + up. Calls still compile, but a `catch (UnknownHostException)` wrapped around one does not, since + the exception is no longer throwable there. Drop the catch — widening it to `IOException` does not + compile either, because a catch clause for a checked exception the body cannot throw is an error + in its own right; only `Exception` and its supertypes are exempt from that rule. + ### 4.19.0.7 #### Cloud private-endpoint support via client routes