From d6af4bb767f026a8fdf7f6116a1c317296551e54 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 23 Jul 2026 23:33:12 -0300 Subject: [PATCH 1/5] Add lock and synchronize allowed sessions --- .../com/cloud/consoleproxy/ConsoleProxy.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index a25abac981b9..de5834cffcd2 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -26,7 +26,6 @@ import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.net.URL; -import java.util.HashSet; import java.util.Hashtable; import java.util.Map; import java.util.Properties; @@ -83,8 +82,10 @@ public class ConsoleProxy { static String encryptorPassword = "Dummy"; static final String[] skipProperties = new String[]{"certificate", "cacertificate", "keystore_password", "privatekey"}; - static Set allowedSessions = new HashSet<>(); + static Set allowedSessions = ConcurrentHashMap.newKeySet(); + private static final Object allowedSessionsLock = new Object(); + // Invoked through reflection public static void addAllowedSession(String sessionUuid) { allowedSessions.add(sessionUuid); } @@ -209,13 +210,15 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console } String sessionUuid = param.getSessionUuid(); - if (allowedSessions.contains(sessionUuid)) { - LOGGER.debug("Acquiring the session " + sessionUuid + " not available for future use"); - allowedSessions.remove(sessionUuid); - } else { - LOGGER.info("Session " + sessionUuid + " has already been used, cannot connect"); - authResult.setSuccess(false); - return authResult; + synchronized (allowedSessionsLock) { + if (allowedSessions.contains(sessionUuid)) { + LOGGER.debug("Acquiring the session " + sessionUuid + " not available for future use"); + allowedSessions.remove(sessionUuid); + } else { + LOGGER.info("Session " + sessionUuid + " has already been used, cannot connect"); + authResult.setSuccess(false); + return authResult; + } } String websocketUrl = param.getWebsocketUrl(); From 2515091b87c3fb92fd7134902f4b6cbe0417f496 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 23 Jul 2026 23:47:04 -0300 Subject: [PATCH 2/5] Fix closing connections to VNC ports on console close --- .../com/cloud/consoleproxy/ConsoleProxy.java | 2 +- .../ConsoleProxyNoVNCHandler.java | 16 +++++++++---- .../consoleproxy/ConsoleProxyNoVncClient.java | 3 +++ .../cloud/consoleproxy/vnc/NoVncClient.java | 22 +++++++++++++++++ .../consoleproxy/vnc/network/NioSocket.java | 24 +++++++++++++++++++ .../vnc/network/NioSocketHandler.java | 1 + .../vnc/network/NioSocketHandlerImpl.java | 7 ++++++ 7 files changed, 70 insertions(+), 5 deletions(-) diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index de5834cffcd2..532c73d2461c 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -628,7 +628,7 @@ public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam par } catch (IOException e) { LOGGER.error("Exception while disconnect session of novnc viewer object: " + viewer, e); } - removeViewer(viewer); + viewer.closeClient(); viewer = new ConsoleProxyNoVncClient(session); viewer.initClient(param); connectionMap.put(clientKey, viewer); diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java index a148b988e40d..dba38d5453ce 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java @@ -175,12 +175,20 @@ private boolean checkSessionSourceIp(final Session session, final String sourceI @OnWebSocketClose public void onClose(Session session, int statusCode, String reason) throws IOException, InterruptedException { - String sessionSourceIp = session.getRemoteAddress().getAddress().getHostAddress(); - logger.debug("Closing WebSocket session [source IP: {}, status code: {}].", sessionSourceIp, statusCode); if (viewer != null) { - ConsoleProxy.removeViewer(viewer); + viewer.closeClient(); + } + String sessionSourceIp = getRemoteAddressSafely(session); + logger.debug("WebSocket session [source IP: {}, status code: {}, reason: {}] closed successfully.", sessionSourceIp, statusCode, reason); + } + + private String getRemoteAddressSafely(Session session) { + try { + return session.getRemoteAddress().getAddress().getHostAddress(); + } catch (Exception e) { + logger.debug("Failed to get remote address from WebSocket session", e); + return "unknown"; } - logger.debug("WebSocket session [source IP: {}, status code: {}] closed successfully.", sessionSourceIp, statusCode); } @OnWebSocketFrame diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java index 36dce8b8554c..b844ab1aaa02 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java @@ -374,6 +374,9 @@ public void closeClient() { this.connectionAlive = false; // Clear buffer reference to allow GC when client disconnects this.readBuffer = null; + if (client != null) { + client.close(); + } ConsoleProxy.removeViewer(this); } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java index ca7577d2bfcb..938b810356be 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java @@ -129,6 +129,28 @@ public void proxyMsgOverWebSocketConnection(ByteBuffer msg) { } } + public void close() { + if (nioSocketConnection != null) { + nioSocketConnection.close(); + } + if (webSocketReverseProxy != null) { + webSocketReverseProxy.close(); + } + if (socket != null) { + try { + if (is != null) { + is.close(); + } + if (os != null) { + os.close(); + } + socket.close(); + } catch (IOException e) { + logger.debug("Error closing socket: " + e.getMessage(), e); + } + } + } + private void setTunnelSocketStreams() throws IOException { this.is = new DataInputStream(this.socket.getInputStream()); this.os = new DataOutputStream(this.socket.getOutputStream()); diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java index 4ab88ea9fc72..60ec0f312353 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java @@ -120,4 +120,28 @@ protected int writeToSocketChannel(ByteBuffer buf, int len) { return 0; } } + + public void close() { + try { + if (socketChannel != null) { + socketChannel.close(); + } + } catch (IOException e) { + logger.debug("Error closing socket channel: " + e.getMessage(), e); + } + try { + if (readSelector != null) { + readSelector.close(); + } + } catch (IOException e) { + logger.debug("Error closing read selector: " + e.getMessage(), e); + } + try { + if (writeSelector != null) { + writeSelector.close(); + } + } catch (IOException e) { + logger.debug("Error closing write selector: " + e.getMessage(), e); + } + } } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java index 757f9c126ec0..02ee2ad7e508 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java @@ -41,4 +41,5 @@ public interface NioSocketHandler { void flushWriteBuffer(); void startTLSConnection(NioSocketSSLEngineManager sslEngineManager); boolean isTLSConnection(); + void close(); } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java index fc19c36b3edd..0472a338a855 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java @@ -28,10 +28,12 @@ public class NioSocketHandlerImpl implements NioSocketHandler { private NioSocketInputStream inputStream; private NioSocketOutputStream outputStream; private boolean isTLS = false; + private final NioSocket socket; protected Logger logger = LogManager.getLogger(getClass()); public NioSocketHandlerImpl(NioSocket socket) { + this.socket = socket; this.inputStream = new NioSocketInputStream(ConsoleProxy.defaultBufferSize, socket); this.outputStream = new NioSocketOutputStream(ConsoleProxy.defaultBufferSize, socket); } @@ -109,4 +111,9 @@ public NioSocketInputStream getInputStream() { public NioSocketOutputStream getOutputStream() { return outputStream; } + + @Override + public void close() { + socket.close(); + } } From f8f185f49c5ec6109de99e19cf6ccb5d1f002599 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 27 Jul 2026 11:17:32 -0300 Subject: [PATCH 3/5] Add reconnection grant window to prevent network connectivity issues after acquiring a session --- .../com/cloud/consoleproxy/ConsoleProxy.java | 61 ++++++++++++++++++- .../consoleproxy/ConsoleProxyGCThread.java | 1 + .../consoleproxy/ConsoleProxyNoVncClient.java | 1 + systemvm/agent/conf/consoleproxy.properties | 1 + 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index 532c73d2461c..daf1122656fe 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -85,11 +85,59 @@ public class ConsoleProxy { static Set allowedSessions = ConcurrentHashMap.newKeySet(); private static final Object allowedSessionsLock = new Object(); + private static final Map sessionReconnectGrants = new ConcurrentHashMap<>(); + static int sessionReconnectGraceSeconds = 1; + // Invoked through reflection public static void addAllowedSession(String sessionUuid) { allowedSessions.add(sessionUuid); } + /** + * Grant a short, single-use window to reconnect with the same session UUID in case of a disconnection. + * The grant is bound to the client IP that was using the session so it cannot be redeemed by another client. + * @param sessionUuid session UUID to grant a reconnect window for + * @param clientIp source IP of the client the session was granted to + */ + public static void grantReconnectWindow(String sessionUuid, String clientIp) { + sessionReconnectGrants.put(sessionUuid, new ReconnectGrant(System.currentTimeMillis() + sessionReconnectGraceSeconds * 1000L, clientIp)); + } + + private static boolean consumeReconnectGrant(String sessionUuid, String clientIp) { + ReconnectGrant grant = sessionReconnectGrants.remove(sessionUuid); + if (grant == null || grant.isExpired(System.currentTimeMillis())) { + return false; + } + if (grant.clientIp != null && !grant.clientIp.equals(clientIp)) { + LOGGER.warn("Rejecting reconnect for session " + sessionUuid + " as it was requested from IP " + + clientIp + " but the reconnect window was granted to IP " + grant.clientIp); + return false; + } + return true; + } + + /** + * Drops expired, unclaimed reconnect grants so sessionReconnectGrants doesn't grow unbounded + * when a client never reconnects after a disconnection. Invoked periodically by {@link ConsoleProxyGCThread}. + */ + static void cleanupExpiredReconnectGrants() { + sessionReconnectGrants.entrySet().removeIf(entry -> entry.getValue().isExpired(System.currentTimeMillis())); + } + + private static final class ReconnectGrant { + final long expiryMillis; + final String clientIp; + + ReconnectGrant(long expiryMillis, String clientIp) { + this.expiryMillis = expiryMillis; + this.clientIp = clientIp; + } + + boolean isExpired(long now) { + return now >= expiryMillis; + } + } + private static void configLog4j() { final ClassLoader loader = Thread.currentThread().getContextClassLoader(); URL configUrl = loader.getResource("/conf/log4j-cloud.xml"); @@ -167,6 +215,12 @@ private static void configProxy(Properties conf) { defaultBufferSize = Integer.parseInt(s); LOGGER.info("Setting defaultBufferSize=" + defaultBufferSize); } + + s = conf.getProperty("consoleproxy.sessionReconnectGraceSeconds"); + if (s != null) { + sessionReconnectGraceSeconds = Integer.parseInt(s); + LOGGER.info("Setting sessionReconnectGraceSeconds=" + sessionReconnectGraceSeconds); + } } public static ConsoleProxyServerFactory getHttpServerFactory() { @@ -211,9 +265,10 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console String sessionUuid = param.getSessionUuid(); synchronized (allowedSessionsLock) { - if (allowedSessions.contains(sessionUuid)) { - LOGGER.debug("Acquiring the session " + sessionUuid + " not available for future use"); - allowedSessions.remove(sessionUuid); + if (allowedSessions.remove(sessionUuid)) { + LOGGER.debug("Acquiring the session " + sessionUuid + " for use"); + } else if (consumeReconnectGrant(sessionUuid, param.getClientIp())) { + LOGGER.info("Reconnecting the session " + sessionUuid + " after a dropped connection"); } else { LOGGER.info("Session " + sessionUuid + " has already been used, cannot connect"); authResult.setSuccess(false); diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyGCThread.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyGCThread.java index 0e8f576cf6db..74d83fb591ff 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyGCThread.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyGCThread.java @@ -75,6 +75,7 @@ public void run() { while (true) { cleanupLogging(); + ConsoleProxy.cleanupExpiredReconnectGrants(); bReportLoad = false; if (logger.isDebugEnabled()) { diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java index b844ab1aaa02..22cd546a78a9 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java @@ -177,6 +177,7 @@ public void run() { } } logger.info("Connection with client [{}] [IP: {}] is dead.", clientId, clientSourceIp); + ConsoleProxy.grantReconnectWindow(sessionUuid, clientSourceIp); } catch (IOException e) { logger.error("Error on VNC client", e); } diff --git a/systemvm/agent/conf/consoleproxy.properties b/systemvm/agent/conf/consoleproxy.properties index 361b0a33a054..6255a6906524 100644 --- a/systemvm/agent/conf/consoleproxy.properties +++ b/systemvm/agent/conf/consoleproxy.properties @@ -22,3 +22,4 @@ consoleproxy.jarDir=./applet/ consoleproxy.viewerLinger=180 consoleproxy.reconnectMaxRetry=5 consoleproxy.defaultBufferSize=65536 +consoleproxy.sessionReconnectGraceSeconds=1 From 28acf1f46c291181c4c2e21cec6d20ce304861f0 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 28 Jul 2026 12:15:03 -0300 Subject: [PATCH 4/5] Introduce zone setting to control the reconnection window for console sessions --- .../consoleproxy/ConsoleProxyManager.java | 3 ++ .../consoleproxy/ConsoleProxyManagerImpl.java | 6 ++- .../com/cloud/consoleproxy/ConsoleProxy.java | 42 ++++++++++++------- .../consoleproxy/ConsoleProxyNoVncClient.java | 2 +- systemvm/agent/conf/consoleproxy.properties | 1 - 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java index 47d1a306e4d9..4e71d78fcbbd 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java @@ -84,6 +84,9 @@ public interface ConsoleProxyManager extends Manager, ConsoleProxyService { ConfigKey ConsoleProxyDisableRpFilter = new ConfigKey<>(Boolean.class, "consoleproxy.disable.rpfilter", "Console Proxy", "true", "disable rp_filter on console proxy VM public interface", true, ConfigKey.Scope.Zone, null); + ConfigKey ConsoleProxySessionReconnectionWindow = new ConfigKey<>(Long.class, "consoleproxy.session.reconnection.window", "Console Proxy", "0", + "Reconnection window (in milliseconds) for client IPs to the same session on console proxy VM", true, ConfigKey.Scope.Zone, null); + ConfigKey ConsoleProxyLaunchMax = new ConfigKey<>(Integer.class, "consoleproxy.launch.max", "Console Proxy", "10", "maximum number of console proxy instances per zone can be launched", false, ConfigKey.Scope.Zone, null); diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 018cead22fe3..a365a51902b9 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1210,6 +1210,10 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl if (Boolean.TRUE.equals(disableRpFilter)) { buf.append(" disable_rp_filter=true"); } + Long sessionReconnectionWindow = ConsoleProxySessionReconnectionWindow.valueIn(datacenterId); + if (sessionReconnectionWindow != null && sessionReconnectionWindow > 0) { + buf.append(" session_reconnection_window=").append(sessionReconnectionWindow); + } String msPublicKey = configurationDao.getValue("ssh.publickey"); buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey)); @@ -1591,7 +1595,7 @@ public ConfigKey[] getConfigKeys() { return new ConfigKey[] {ConsoleProxySslEnabled, NoVncConsoleDefault, NoVncConsoleSourceIpCheckEnabled, ConsoleProxyServiceOffering, ConsoleProxyCapacityStandby, ConsoleProxyCapacityScanInterval, ConsoleProxyRestart, ConsoleProxyUrlDomain, ConsoleProxySessionMax, ConsoleProxySessionTimeout, ConsoleProxyDisableRpFilter, ConsoleProxyLaunchMax, ConsoleProxyManagementLastState, ConsoleProxyServiceManagementState, NoVncConsoleShowDot, - ConsoleProxyVmUserData}; + ConsoleProxyVmUserData, ConsoleProxySessionReconnectionWindow}; } protected ConsoleProxyStatus parseJsonToConsoleProxyStatus(String json) throws JsonParseException { diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index daf1122656fe..0cb1c4805dac 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -86,7 +86,7 @@ public class ConsoleProxy { private static final Object allowedSessionsLock = new Object(); private static final Map sessionReconnectGrants = new ConcurrentHashMap<>(); - static int sessionReconnectGraceSeconds = 1; + private static long sessionReconnectionWindowMs = 0L; // Invoked through reflection public static void addAllowedSession(String sessionUuid) { @@ -94,23 +94,34 @@ public static void addAllowedSession(String sessionUuid) { } /** - * Grant a short, single-use window to reconnect with the same session UUID in case of a disconnection. + * Grant the client IP a reconnection window of #{@link #sessionReconnectionWindowMs} ms to the same session UUID in case of a disconnection. * The grant is bound to the client IP that was using the session so it cannot be redeemed by another client. * @param sessionUuid session UUID to grant a reconnect window for * @param clientIp source IP of the client the session was granted to */ - public static void grantReconnectWindow(String sessionUuid, String clientIp) { - sessionReconnectGrants.put(sessionUuid, new ReconnectGrant(System.currentTimeMillis() + sessionReconnectGraceSeconds * 1000L, clientIp)); + public static void grantReconnectWindowForSessionAndClientIp(String sessionUuid, String clientIp) { + if (sessionReconnectionWindowMs > 0) { + ReconnectGrant grant = new ReconnectGrant(System.currentTimeMillis() + sessionReconnectionWindowMs, clientIp); + sessionReconnectGrants.put(sessionUuid, grant); + } } - private static boolean consumeReconnectGrant(String sessionUuid, String clientIp) { + /** + * True if the session UUID has been granted reconnection, within the reconnection window #{@link #sessionReconnectionWindowMs}. + */ + private static boolean isSessionReconnectionGrantedForClientIp(String sessionUuid, String clientIp) { ReconnectGrant grant = sessionReconnectGrants.remove(sessionUuid); - if (grant == null || grant.isExpired(System.currentTimeMillis())) { + if (grant == null) { + return false; + } + if (grant.isExpired(System.currentTimeMillis())) { + LOGGER.warn("Rejecting reconnection for session {} as the reconnect window: {}ms is already expired", + sessionUuid, sessionReconnectionWindowMs); return false; } if (grant.clientIp != null && !grant.clientIp.equals(clientIp)) { - LOGGER.warn("Rejecting reconnect for session " + sessionUuid + " as it was requested from IP " + - clientIp + " but the reconnect window was granted to IP " + grant.clientIp); + LOGGER.warn("Rejecting reconnection for session {} as it was requested from IP {} " + + "but the session was granted to IP {}", sessionUuid, clientIp, grant.clientIp); return false; } return true; @@ -216,10 +227,10 @@ private static void configProxy(Properties conf) { LOGGER.info("Setting defaultBufferSize=" + defaultBufferSize); } - s = conf.getProperty("consoleproxy.sessionReconnectGraceSeconds"); + s = conf.getProperty("session_reconnection_window"); if (s != null) { - sessionReconnectGraceSeconds = Integer.parseInt(s); - LOGGER.info("Setting sessionReconnectGraceSeconds=" + sessionReconnectGraceSeconds); + sessionReconnectionWindowMs = Long.parseLong(s); + LOGGER.info("Setting sessionReconnectionWindowMs=" + sessionReconnectionWindowMs); } } @@ -266,11 +277,12 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console String sessionUuid = param.getSessionUuid(); synchronized (allowedSessionsLock) { if (allowedSessions.remove(sessionUuid)) { - LOGGER.debug("Acquiring the session " + sessionUuid + " for use"); - } else if (consumeReconnectGrant(sessionUuid, param.getClientIp())) { - LOGGER.info("Reconnecting the session " + sessionUuid + " after a dropped connection"); + LOGGER.debug("Acquiring the session {} from client IP {}", sessionUuid, param.getClientIp()); + } else if (isSessionReconnectionGrantedForClientIp(sessionUuid, param.getClientIp())) { + LOGGER.info("Reconnecting the session {} after a dropped connection", sessionUuid); + return authResult; } else { - LOGGER.info("Session " + sessionUuid + " has already been used, cannot connect"); + LOGGER.info("Invalid or already used session {}, cannot connect", sessionUuid); authResult.setSuccess(false); return authResult; } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java index 22cd546a78a9..2ac85e5c5d33 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java @@ -177,7 +177,7 @@ public void run() { } } logger.info("Connection with client [{}] [IP: {}] is dead.", clientId, clientSourceIp); - ConsoleProxy.grantReconnectWindow(sessionUuid, clientSourceIp); + ConsoleProxy.grantReconnectWindowForSessionAndClientIp(sessionUuid, clientSourceIp); } catch (IOException e) { logger.error("Error on VNC client", e); } diff --git a/systemvm/agent/conf/consoleproxy.properties b/systemvm/agent/conf/consoleproxy.properties index 6255a6906524..361b0a33a054 100644 --- a/systemvm/agent/conf/consoleproxy.properties +++ b/systemvm/agent/conf/consoleproxy.properties @@ -22,4 +22,3 @@ consoleproxy.jarDir=./applet/ consoleproxy.viewerLinger=180 consoleproxy.reconnectMaxRetry=5 consoleproxy.defaultBufferSize=65536 -consoleproxy.sessionReconnectGraceSeconds=1 From d1e686894370c2c371cca3b6da51d7b1ffbd3b61 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 17 Sep 2026 15:43:29 -0300 Subject: [PATCH 5/5] Increase logs for intermittent issues troubleshooting --- .../com/cloud/consoleproxy/ConsoleProxy.java | 26 ++++++++++++------- .../ConsoleProxyNoVNCHandler.java | 9 ++++++- .../cloud/consoleproxy/vnc/NoVncClient.java | 26 ++++++++++++++----- .../consoleproxy/vnc/network/NioSocket.java | 10 ++++--- .../vnc/network/NioSocketHandlerImpl.java | 14 ++++++++++ systemvm/agent/noVNC/app/ui.js | 1 - systemvm/agent/noVNC/core/rfb.js | 2 +- systemvm/agent/noVNC/core/websock.js | 1 + 8 files changed, 68 insertions(+), 21 deletions(-) diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index 0cb1c4805dac..251a4c92c4cf 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -112,15 +112,17 @@ public static void grantReconnectWindowForSessionAndClientIp(String sessionUuid, private static boolean isSessionReconnectionGrantedForClientIp(String sessionUuid, String clientIp) { ReconnectGrant grant = sessionReconnectGrants.remove(sessionUuid); if (grant == null) { + LOGGER.debug("No reconnect grant found for session {} requested from IP {} (reconnection window: {}ms)", + sessionUuid, clientIp, sessionReconnectionWindowMs); return false; } if (grant.isExpired(System.currentTimeMillis())) { - LOGGER.warn("Rejecting reconnection for session {} as the reconnect window: {}ms is already expired", + LOGGER.debug("Rejecting reconnection for session {} as the reconnect window: {}ms is already expired", sessionUuid, sessionReconnectionWindowMs); return false; } if (grant.clientIp != null && !grant.clientIp.equals(clientIp)) { - LOGGER.warn("Rejecting reconnection for session {} as it was requested from IP {} " + + LOGGER.debug("Rejecting reconnection for session {} as it was requested from IP {} " + "but the session was granted to IP {}", sessionUuid, clientIp, grant.clientIp); return false; } @@ -249,7 +251,7 @@ public static ConsoleProxyServerFactory getHttpServerFactory() { return null; } } catch (ClassNotFoundException e) { - LOGGER.warn("Unable to find http server factory class: " + factoryClzName); + LOGGER.debug("Unable to find http server factory class: " + factoryClzName); return new ConsoleProxyBaseServerFactoryImpl(); } } @@ -275,6 +277,8 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console } String sessionUuid = param.getSessionUuid(); + LOGGER.debug("Authenticating console access [session: {}, clientIp: {}, reauthentication: {}, currently allowed sessions: {}]", + sessionUuid, param.getClientIp(), reauthentication, allowedSessions.size()); synchronized (allowedSessionsLock) { if (allowedSessions.remove(sessionUuid)) { LOGGER.debug("Acquiring the session {} from client IP {}", sessionUuid, param.getClientIp()); @@ -282,7 +286,11 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console LOGGER.info("Reconnecting the session {} after a dropped connection", sessionUuid); return authResult; } else { - LOGGER.info("Invalid or already used session {}, cannot connect", sessionUuid); + LOGGER.debug("Rejecting console session {} from client IP {}: session is not in the allowed set (already " + + "consumed, unknown, or expired) and no valid reconnect grant is available " + + "(reconnection window: {}ms{})", + sessionUuid, param.getClientIp(), sessionReconnectionWindowMs, + sessionReconnectionWindowMs <= 0 ? ", reconnection disabled" : ""); authResult.setSuccess(false); return authResult; } @@ -320,7 +328,7 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console authResult.setSuccess(false); } } else { - LOGGER.warn("Private channel towards management server is not setup. Switch to offline mode and allow access to vm: " + param.getClientTag()); + LOGGER.debug("Private channel towards management server is not setup. Switch to offline mode and allow access to vm: " + param.getClientTag()); } return authResult; @@ -336,7 +344,7 @@ public static void reportLoadInfo(String gsonLoadInfo) { LOGGER.error("Unable to invoke reportLoadInfo due to " + e.getMessage()); } } else { - LOGGER.warn("Private channel towards management server is not setup. Switch to offline mode and ignore load report"); + LOGGER.debug("Private channel towards management server is not setup. Switch to offline mode and ignore load report"); } } @@ -350,7 +358,7 @@ public static void ensureRoute(String address) { LOGGER.error("Unable to invoke ensureRoute due to " + e.getMessage()); } } else { - LOGGER.warn("Unable to find ensureRoute method, console proxy agent is not up to date"); + LOGGER.debug("Unable to find ensureRoute method, console proxy agent is not up to date"); } } @@ -542,7 +550,7 @@ public static ConsoleProxyClient getVncViewer(ConsoleProxyClientParam param) thr LOGGER.info("The rfb thread died, reinitializing the viewer " + viewer); viewer.initClient(param); } else if (!param.getClientHostPassword().equals(viewer.getClientHostPassword())) { - LOGGER.warn("Bad sid detected(VNC port may be reused). sid in session: " + viewer.getClientHostPassword() + ", sid in request: " + + LOGGER.debug("Bad sid detected(VNC port may be reused). sid in session: " + viewer.getClientHostPassword() + ", sid in request: " + param.getClientHostPassword()); viewer.initClient(param); } @@ -634,7 +642,7 @@ public static void authenticationExternally(ConsoleProxyClientParam param) throw ConsoleProxyAuthenticationResult authResult = authenticateConsoleAccess(param, false); if (authResult == null || !authResult.isSuccess()) { - LOGGER.warn("External authenticator failed authentication request for vm " + param.getClientTag() + " with sid " + param.getClientHostPassword()); + LOGGER.debug("External authenticator failed authentication request for vm " + param.getClientTag() + " with sid " + param.getClientHostPassword()); throw new AuthenticationException("External authenticator failed request for vm " + param.getClientTag() + " with sid " + param.getClientHostPassword()); } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java index dba38d5453ce..04f08a7f2c73 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java @@ -57,6 +57,8 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques throws IOException, ServletException { if (this.getWebSocketFactory().isUpgradeRequest(request, response)) { + logger.debug("Received WebSocket upgrade request [path: {}, query: {}, remote IP: {}]", + target, request.getQueryString(), request.getRemoteAddr()); response.addHeader("Sec-WebSocket-Protocol", "binary"); if (this.getWebSocketFactory().acceptWebSocket(request, response)) { baseRequest.setHandled(true); @@ -95,6 +97,9 @@ public void onConnect(final Session session) throws IOException, InterruptedExce String clientIp = session.getRemoteAddress().getAddress().getHostAddress(); boolean sessionRequiresNewViewer = Boolean.parseBoolean(queryMap.get("sessionRequiresNewViewer")); + logger.info("WebSocket connect attempt [session UUID: {}, client IP: {}, host: {}, port: {}, sessionRequiresNewViewer: {}]", + sessionUuid, clientIp, host, portStr, sessionRequiresNewViewer); + if (tag == null) tag = ""; @@ -199,6 +204,8 @@ public void onFrame(Frame f) throws IOException { @OnWebSocketError public void onError(Throwable cause) { - logger.error("Error on WebSocket [client ID: {}, session UUID: {}].", cause, viewer.getClientId(), viewer.getSessionUuid()); + String clientId = viewer != null ? String.valueOf(viewer.getClientId()) : "unknown (no viewer created)"; + String sessionUuid = viewer != null ? viewer.getSessionUuid() : "unknown"; + logger.error("Error on WebSocket [client ID: {}, session UUID: {}]: {}", clientId, sessionUuid, cause.getMessage(), cause); } } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java index 938b810356be..e49bfa02fc43 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java @@ -413,13 +413,26 @@ public ByteBuffer handshakeProtocolVersion() { return verStr; } + protected static final long WAIT_FOR_NOVNC_TIMEOUT_MS = 30000; + public void waitForNoVNCReply() { - int cycles = 0; - while (isWaitForNoVnc()) { - cycles++; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Waited %d cycles for NoVnc", cycles)); + long start = System.currentTimeMillis(); + logger.debug("Waiting for a reply from the noVNC client during handshake"); + synchronized (lock) { + while (waitForNoVnc) { + long remaining = WAIT_FOR_NOVNC_TIMEOUT_MS - (System.currentTimeMillis() - start); + if (remaining <= 0) { + logger.debug("Timed out after {} ms waiting for a reply from the noVNC client during handshake", WAIT_FOR_NOVNC_TIMEOUT_MS); + break; + } + try { + lock.wait(remaining); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Interrupted while waiting for a reply from the noVNC client during handshake", e); + break; + } + } } } @@ -468,6 +481,7 @@ public int handshakeSecurityType() { public void setWaitForNoVnc(boolean val) { synchronized (lock) { this.waitForNoVnc = val; + lock.notifyAll(); } } diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java index 60ec0f312353..a0c94d7e3558 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java @@ -53,9 +53,13 @@ private void initializeSocket() throws IOException { } } - private void waitForSocketSelectorConnected(Selector selector) throws IOException { + private void waitForSocketSelectorConnected(Selector selector, String host, int port) throws IOException { + long startTime = System.currentTimeMillis(); + int attempts = 0; while (selector.select(CONNECTION_TIMEOUT_MILLIS) <= 0) { - logger.debug("Waiting for ready operations to connect to the socket"); + attempts++; + logger.debug("Still waiting to establish VNC backend connection to {}:{} after {} ms ({} attempt(s) of {} ms each)", + host, port, System.currentTimeMillis() - startTime, attempts, CONNECTION_TIMEOUT_MILLIS); } Set keys = selector.selectedKeys(); for (SelectionKey selectionKey: keys) { @@ -75,7 +79,7 @@ private void connectSocket(String host, int port) throws IOException { Selector selector = Selector.open(); socketChannel.register(selector, SelectionKey.OP_CONNECT); - waitForSocketSelectorConnected(selector); + waitForSocketSelectorConnected(selector, host, port); } catch (IOException e) { logger.error("Error connecting NioSocket to {}:{}: {}", host, port, e.getMessage(), e); throw e; diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java index 0472a338a855..fe1494ec7ba0 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java @@ -53,10 +53,24 @@ public void readBytes(ByteBuffer data, int length) { inputStream.readBytes(data, length); } + private static final long STALL_WARNING_INTERVAL_NANOS = java.util.concurrent.TimeUnit.SECONDS.toNanos(5); + @Override public void waitForBytesAvailableForReading(int bytes) { + long startTime = System.nanoTime(); + long lastLogTime = startTime; + long cycles = 0; while (!inputStream.checkForSizeWithoutWait(bytes)) { logger.trace("Waiting for inStream to be ready"); + cycles++; + if (cycles % 1_000_000 == 0) { + long now = System.nanoTime(); + if (now - lastLogTime >= STALL_WARNING_INTERVAL_NANOS) { + logger.debug("Still waiting for {} byte(s) from the VNC backend socket after {} ms", + bytes, java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(now - startTime)); + lastLogTime = now; + } + } } } diff --git a/systemvm/agent/noVNC/app/ui.js b/systemvm/agent/noVNC/app/ui.js index 85530549e604..61901b875001 100644 --- a/systemvm/agent/noVNC/app/ui.js +++ b/systemvm/agent/noVNC/app/ui.js @@ -1124,7 +1124,6 @@ const UI = { url.protocol = (window.location.protocol === "https:") ? 'wss:' : 'ws:'; } - url.href += '/' + path; url.href += '?token=' + token; if (extra) { diff --git a/systemvm/agent/noVNC/core/rfb.js b/systemvm/agent/noVNC/core/rfb.js index 59218b136b94..8913c2a35824 100644 --- a/systemvm/agent/noVNC/core/rfb.js +++ b/systemvm/agent/noVNC/core/rfb.js @@ -844,7 +844,7 @@ export default class RFB extends EventTargetMixin { } _socketError(e) { - Log.Warn("WebSocket on-error event"); + Log.Warn("WebSocket on-error event: " + e); } _focusCanvas(event) { diff --git a/systemvm/agent/noVNC/core/websock.js b/systemvm/agent/noVNC/core/websock.js index ae17a4409811..5ce5e7a644b0 100644 --- a/systemvm/agent/noVNC/core/websock.js +++ b/systemvm/agent/noVNC/core/websock.js @@ -279,6 +279,7 @@ export default class Websock { this._websocket.onclose = (e) => { Log.Debug(">> WebSock.onclose"); + Log.Warn("WebSocket onclose event: " + e); this._eventHandlers.close(e); Log.Debug("<< WebSock.onclose"); };