From 60d6b6d5c1fdeab9ac079d8d63f61b393477f0b2 Mon Sep 17 00:00:00 2001 From: nicholaspai Date: Mon, 17 Aug 2026 22:50:26 -0400 Subject: [PATCH 1/2] improve(serverless-orchestration): let bots opt out of hub paging The hub logs every spoke failure at `error`, which the PagerDuty transport picks up and pages on. A single testnet bot failing in a shared config file therefore pages the on-call for the whole across-bots service. A bot config can now set `pageOnError: false`. When every failing bot in a run carries the flag the hub logs at `warn` instead: Slack and GCP logging still receive it, PagerDuty does not. Any run with an unflagged failing bot, or a failure the hub cannot attribute to a bot, pages exactly as before. Co-Authored-By: Claude Opus 5 --- .../src/ServerlessHub.js | 9 ++- .../test/ServerlessHub.js | 72 +++++++++++++++---- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/packages/serverless-orchestration/src/ServerlessHub.js b/packages/serverless-orchestration/src/ServerlessHub.js index d3381eacb8..c412d817a3 100644 --- a/packages/serverless-orchestration/src/ServerlessHub.js +++ b/packages/serverless-orchestration/src/ServerlessHub.js @@ -102,6 +102,7 @@ hub.post("/", async (req, res) => { // Use a custom logger if provided. Otherwise, initialize a local logger. // Note: no reason to put this into the try-catch since a logger is required to throw the error. const logger = customLogger || createNewLogger(); + let configObject; // Hoisted so the catch block can read per-bot pageOnError. try { logger.debug({ at: "ServerlessHub", message: "Running Serverless hub query", reqBody: req.body, hubConfig }); @@ -115,7 +116,7 @@ hub.post("/", async (req, res) => { req.body.rejectSpokeDelay !== undefined ? parseInt(req.body.rejectSpokeDelay) : hubConfig.rejectSpokeDelay; // Get the config file from the GCP bucket if running in production mode. Else, pull the config from env. - const configObject = await _fetchConfig(req.body.bucket, req.body.configFile); + configObject = await _fetchConfig(req.body.bucket, req.body.configFile); if (!configObject) throw new Error( `Serverless hub missing a config object! GCPBucket:${req.body.bucket} configFile:${req.body.configFile}` @@ -348,7 +349,11 @@ hub.post("/", async (req, res) => { message: "Some spoke calls returned errors (details)🚨", output: errorOutput, }); - logger.error({ + // The PagerDuty transport only accepts `error`; `warn` still reaches Slack and GCP logging. Failures we + // can't attribute to a bot page anyway. + const failedBots = Object.keys(errorOutput?.errorOutputs ?? {}); + const pages = !failedBots.length || failedBots.some((bot) => configObject?.[bot]?.pageOnError !== false); + logger[pages ? "error" : "warn"]({ at: "ServerlessHub", message: "Some spoke calls returned errors 🚨", retriedSpokes: errorOutput.retriedOutputs, diff --git a/packages/serverless-orchestration/test/ServerlessHub.js b/packages/serverless-orchestration/test/ServerlessHub.js index edaf79332a..97f4d5ce12 100644 --- a/packages/serverless-orchestration/test/ServerlessHub.js +++ b/packages/serverless-orchestration/test/ServerlessHub.js @@ -175,10 +175,7 @@ describe("ServerlessHub.js", function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor - const defaultConfig = { - serverlessCommand: "true", - environmentVariables: { CUSTOM_NODE_URL: network.config.url }, - }; + const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } }; const hubConfig = { // no named spoke testDefaultInstance: defaultConfig, @@ -204,10 +201,7 @@ describe("ServerlessHub.js", function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor - const defaultConfig = { - serverlessCommand: "true", - environmentVariables: { CUSTOM_NODE_URL: network.config.url }, - }; + const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } }; const hubConfig = { testInvalidInstance: { ...defaultConfig, spokeUrlName: "invalid" } }; // Set env variables for the hub to pull from. Add the startingBlockNumber and the hubConfig. setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); @@ -302,6 +296,63 @@ describe("ServerlessHub.js", function () { timeoutSpokeInstance.close(); }); + it("ServerlessHub does not page when every failing bot sets pageOnError: false", async function () { + const testBucket = "test-bucket"; // name of the config bucket. + const testConfigFile = "test-config-file"; // name of the config file. + const startingBlockNumber = Number(await provider.getBlockNumber()); + + const hubConfig = { + testServerlessMonitor: { + serverlessCommand: "true", + pageOnError: false, + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + }; + setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); + setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig)); + + const testHubPort = 8085; // create a separate port to run this specific test on. + // Point the hub at a port nothing is listening on to force the spoke call to reject. + await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url); + + const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort); + + // The failure is still reported in full, just below the level the PagerDuty transport accepts. + assert.equal(lastSpyLogLevel(hubSpy), "warn"); + assert.equal(rejectedResponse.res.statusCode, 500); + assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors")); + assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitor")); + }); + it("ServerlessHub still pages when a bot without pageOnError: false fails alongside one with it", async function () { + const testBucket = "test-bucket"; // name of the config bucket. + const testConfigFile = "test-config-file"; // name of the config file. + const startingBlockNumber = Number(await provider.getBlockNumber()); + + const hubConfig = { + testServerlessMonitorNoPage: { + serverlessCommand: "true", + pageOnError: false, + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + testServerlessMonitorPages: { + serverlessCommand: "true", + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + }; + setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); + setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig)); + + const testHubPort = 8086; // create a separate port to run this specific test on. + // Point the hub at a port nothing is listening on to force both spoke calls to reject. + await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url); + + const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort); + + assert.equal(lastSpyLogLevel(hubSpy), "error"); + assert.equal(rejectedResponse.res.statusCode, 500); + assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors")); + assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitorPages")); + }); it("ServerlessHub can correctly execute multiple bots in parallel", async function () { // Set up the environment for testing. For these tests the hub is tested in `localStorage` mode where it will // read in hub configs and previous block numbers from the local storage of machine. This execution mode would be @@ -593,10 +644,7 @@ describe("ServerlessHub.js", function () { // Logs should include correct starting and latest block numbers for the alternate network. const alternateBlockNumbers = { - [alternateChainId]: { - lastQueriedBlockNumber, - latestBlockNumber: latestAlternateBlockNumber, - }, + [alternateChainId]: { lastQueriedBlockNumber, latestBlockNumber: latestAlternateBlockNumber }, }; // Strip enclosing curly braces as there are also other items in the logged object. From c2d66bca07f5c20b98d7746ecca2bdc7e6016f53 Mon Sep 17 00:00:00 2001 From: nicholaspai Date: Tue, 18 Aug 2026 07:53:40 -0400 Subject: [PATCH 2/2] improve(serverless-orchestration): rename pageOnError to hubPageOnFailure pageOnError read as though it controlled the bot's own PagerDuty transport, which is a separate flag inside environmentVariables. The name now says which pager it flips, and "failure" matches the condition the hub actually reacts to (spoke timeout, rejection, non-zero exit, empty stdout). Co-Authored-By: Claude Opus 5 Signed-off-by: nicholaspai --- packages/serverless-orchestration/src/ServerlessHub.js | 4 ++-- packages/serverless-orchestration/test/ServerlessHub.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/serverless-orchestration/src/ServerlessHub.js b/packages/serverless-orchestration/src/ServerlessHub.js index c412d817a3..c6014807eb 100644 --- a/packages/serverless-orchestration/src/ServerlessHub.js +++ b/packages/serverless-orchestration/src/ServerlessHub.js @@ -102,7 +102,7 @@ hub.post("/", async (req, res) => { // Use a custom logger if provided. Otherwise, initialize a local logger. // Note: no reason to put this into the try-catch since a logger is required to throw the error. const logger = customLogger || createNewLogger(); - let configObject; // Hoisted so the catch block can read per-bot pageOnError. + let configObject; // Hoisted so the catch block can read per-bot hubPageOnFailure. try { logger.debug({ at: "ServerlessHub", message: "Running Serverless hub query", reqBody: req.body, hubConfig }); @@ -352,7 +352,7 @@ hub.post("/", async (req, res) => { // The PagerDuty transport only accepts `error`; `warn` still reaches Slack and GCP logging. Failures we // can't attribute to a bot page anyway. const failedBots = Object.keys(errorOutput?.errorOutputs ?? {}); - const pages = !failedBots.length || failedBots.some((bot) => configObject?.[bot]?.pageOnError !== false); + const pages = !failedBots.length || failedBots.some((bot) => configObject?.[bot]?.hubPageOnFailure !== false); logger[pages ? "error" : "warn"]({ at: "ServerlessHub", message: "Some spoke calls returned errors 🚨", diff --git a/packages/serverless-orchestration/test/ServerlessHub.js b/packages/serverless-orchestration/test/ServerlessHub.js index 97f4d5ce12..0347493f6f 100644 --- a/packages/serverless-orchestration/test/ServerlessHub.js +++ b/packages/serverless-orchestration/test/ServerlessHub.js @@ -296,7 +296,7 @@ describe("ServerlessHub.js", function () { timeoutSpokeInstance.close(); }); - it("ServerlessHub does not page when every failing bot sets pageOnError: false", async function () { + it("ServerlessHub does not page when every failing bot sets hubPageOnFailure: false", async function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); @@ -304,7 +304,7 @@ describe("ServerlessHub.js", function () { const hubConfig = { testServerlessMonitor: { serverlessCommand: "true", - pageOnError: false, + hubPageOnFailure: false, environmentVariables: { CUSTOM_NODE_URL: network.config.url }, }, }; @@ -323,7 +323,7 @@ describe("ServerlessHub.js", function () { assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors")); assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitor")); }); - it("ServerlessHub still pages when a bot without pageOnError: false fails alongside one with it", async function () { + it("ServerlessHub still pages when a bot without hubPageOnFailure: false fails alongside one with it", async function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); @@ -331,7 +331,7 @@ describe("ServerlessHub.js", function () { const hubConfig = { testServerlessMonitorNoPage: { serverlessCommand: "true", - pageOnError: false, + hubPageOnFailure: false, environmentVariables: { CUSTOM_NODE_URL: network.config.url }, }, testServerlessMonitorPages: {