diff --git a/packages/serverless-orchestration/src/ServerlessHub.js b/packages/serverless-orchestration/src/ServerlessHub.js index d3381eacb8..c6014807eb 100644 --- a/packages/serverless-orchestration/src/ServerlessHub.js +++ b/packages/serverless-orchestration/src/ServerlessHub.js @@ -102,6 +102,7 @@ hub.post("/", async (req, res) => { // Use a custom logger if provided. Otherwise, initialize a local logger. // Note: no reason to put this into the try-catch since a logger is required to throw the error. const logger = customLogger || createNewLogger(); + let configObject; // Hoisted so the catch block can read per-bot hubPageOnFailure. try { logger.debug({ at: "ServerlessHub", message: "Running Serverless hub query", reqBody: req.body, hubConfig }); @@ -115,7 +116,7 @@ hub.post("/", async (req, res) => { req.body.rejectSpokeDelay !== undefined ? parseInt(req.body.rejectSpokeDelay) : hubConfig.rejectSpokeDelay; // Get the config file from the GCP bucket if running in production mode. Else, pull the config from env. - const configObject = await _fetchConfig(req.body.bucket, req.body.configFile); + configObject = await _fetchConfig(req.body.bucket, req.body.configFile); if (!configObject) throw new Error( `Serverless hub missing a config object! GCPBucket:${req.body.bucket} configFile:${req.body.configFile}` @@ -348,7 +349,11 @@ hub.post("/", async (req, res) => { message: "Some spoke calls returned errors (details)🚨", output: errorOutput, }); - logger.error({ + // The PagerDuty transport only accepts `error`; `warn` still reaches Slack and GCP logging. Failures we + // can't attribute to a bot page anyway. + const failedBots = Object.keys(errorOutput?.errorOutputs ?? {}); + const pages = !failedBots.length || failedBots.some((bot) => configObject?.[bot]?.hubPageOnFailure !== false); + logger[pages ? "error" : "warn"]({ at: "ServerlessHub", message: "Some spoke calls returned errors 🚨", retriedSpokes: errorOutput.retriedOutputs, diff --git a/packages/serverless-orchestration/test/ServerlessHub.js b/packages/serverless-orchestration/test/ServerlessHub.js index edaf79332a..0347493f6f 100644 --- a/packages/serverless-orchestration/test/ServerlessHub.js +++ b/packages/serverless-orchestration/test/ServerlessHub.js @@ -175,10 +175,7 @@ describe("ServerlessHub.js", function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor - const defaultConfig = { - serverlessCommand: "true", - environmentVariables: { CUSTOM_NODE_URL: network.config.url }, - }; + const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } }; const hubConfig = { // no named spoke testDefaultInstance: defaultConfig, @@ -204,10 +201,7 @@ describe("ServerlessHub.js", function () { const testBucket = "test-bucket"; // name of the config bucket. const testConfigFile = "test-config-file"; // name of the config file. const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor - const defaultConfig = { - serverlessCommand: "true", - environmentVariables: { CUSTOM_NODE_URL: network.config.url }, - }; + const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } }; const hubConfig = { testInvalidInstance: { ...defaultConfig, spokeUrlName: "invalid" } }; // Set env variables for the hub to pull from. Add the startingBlockNumber and the hubConfig. setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); @@ -302,6 +296,63 @@ describe("ServerlessHub.js", function () { timeoutSpokeInstance.close(); }); + it("ServerlessHub does not page when every failing bot sets hubPageOnFailure: false", async function () { + const testBucket = "test-bucket"; // name of the config bucket. + const testConfigFile = "test-config-file"; // name of the config file. + const startingBlockNumber = Number(await provider.getBlockNumber()); + + const hubConfig = { + testServerlessMonitor: { + serverlessCommand: "true", + hubPageOnFailure: false, + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + }; + setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); + setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig)); + + const testHubPort = 8085; // create a separate port to run this specific test on. + // Point the hub at a port nothing is listening on to force the spoke call to reject. + await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url); + + const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort); + + // The failure is still reported in full, just below the level the PagerDuty transport accepts. + assert.equal(lastSpyLogLevel(hubSpy), "warn"); + assert.equal(rejectedResponse.res.statusCode, 500); + assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors")); + assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitor")); + }); + it("ServerlessHub still pages when a bot without hubPageOnFailure: false fails alongside one with it", async function () { + const testBucket = "test-bucket"; // name of the config bucket. + const testConfigFile = "test-config-file"; // name of the config file. + const startingBlockNumber = Number(await provider.getBlockNumber()); + + const hubConfig = { + testServerlessMonitorNoPage: { + serverlessCommand: "true", + hubPageOnFailure: false, + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + testServerlessMonitorPages: { + serverlessCommand: "true", + environmentVariables: { CUSTOM_NODE_URL: network.config.url }, + }, + }; + setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber); + setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig)); + + const testHubPort = 8086; // create a separate port to run this specific test on. + // Point the hub at a port nothing is listening on to force both spoke calls to reject. + await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url); + + const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort); + + assert.equal(lastSpyLogLevel(hubSpy), "error"); + assert.equal(rejectedResponse.res.statusCode, 500); + assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors")); + assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitorPages")); + }); it("ServerlessHub can correctly execute multiple bots in parallel", async function () { // Set up the environment for testing. For these tests the hub is tested in `localStorage` mode where it will // read in hub configs and previous block numbers from the local storage of machine. This execution mode would be @@ -593,10 +644,7 @@ describe("ServerlessHub.js", function () { // Logs should include correct starting and latest block numbers for the alternate network. const alternateBlockNumbers = { - [alternateChainId]: { - lastQueriedBlockNumber, - latestBlockNumber: latestAlternateBlockNumber, - }, + [alternateChainId]: { lastQueriedBlockNumber, latestBlockNumber: latestAlternateBlockNumber }, }; // Strip enclosing curly braces as there are also other items in the logged object.