improve(serverless-orchestration): let bots opt out of hub paging

packages/serverless-orchestration/src/ServerlessHub.js

@@ -102,6 +102,7 @@ hub.post("/", async (req, res) => {
   // Use a custom logger if provided. Otherwise, initialize a local logger.
   // Note: no reason to put this into the try-catch since a logger is required to throw the error.
   const logger = customLogger || createNewLogger();
+  let configObject; // Hoisted so the catch block can read per-bot hubPageOnFailure.
   try {
     logger.debug({ at: "ServerlessHub", message: "Running Serverless hub query", reqBody: req.body, hubConfig });
 
@@ -115,7 +116,7 @@ hub.post("/", async (req, res) => {
       req.body.rejectSpokeDelay !== undefined ? parseInt(req.body.rejectSpokeDelay) : hubConfig.rejectSpokeDelay;
 
     // Get the config file from the GCP bucket if running in production mode. Else, pull the config from env.
-    const configObject = await _fetchConfig(req.body.bucket, req.body.configFile);
+    configObject = await _fetchConfig(req.body.bucket, req.body.configFile);
     if (!configObject)
       throw new Error(
         `Serverless hub missing a config object! GCPBucket:${req.body.bucket} configFile:${req.body.configFile}`
@@ -348,7 +349,11 @@ hub.post("/", async (req, res) => {
         message: "Some spoke calls returned errors (details)🚨",
         output: errorOutput,
       });
-      logger.error({
+      // The PagerDuty transport only accepts `error`; `warn` still reaches Slack and GCP logging. Failures we
+      // can't attribute to a bot page anyway.
+      const failedBots = Object.keys(errorOutput?.errorOutputs ?? {});
+      const pages = !failedBots.length || failedBots.some((bot) => configObject?.[bot]?.hubPageOnFailure !== false);
+      logger[pages ? "error" : "warn"]({
         at: "ServerlessHub",
         message: "Some spoke calls returned errors 🚨",
         retriedSpokes: errorOutput.retriedOutputs,

packages/serverless-orchestration/test/ServerlessHub.js

@@ -175,10 +175,7 @@ describe("ServerlessHub.js", function () {
     const testBucket = "test-bucket"; // name of the config bucket.
     const testConfigFile = "test-config-file"; // name of the config file.
     const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor
-    const defaultConfig = {
-      serverlessCommand: "true",
-      environmentVariables: { CUSTOM_NODE_URL: network.config.url },
-    };
+    const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } };
     const hubConfig = {
       // no named spoke
       testDefaultInstance: defaultConfig,
@@ -204,10 +201,7 @@ describe("ServerlessHub.js", function () {
     const testBucket = "test-bucket"; // name of the config bucket.
     const testConfigFile = "test-config-file"; // name of the config file.
     const startingBlockNumber = Number(await provider.getBlockNumber()); // block number to search from for monitor
-    const defaultConfig = {
-      serverlessCommand: "true",
-      environmentVariables: { CUSTOM_NODE_URL: network.config.url },
-    };
+    const defaultConfig = { serverlessCommand: "true", environmentVariables: { CUSTOM_NODE_URL: network.config.url } };
     const hubConfig = { testInvalidInstance: { ...defaultConfig, spokeUrlName: "invalid" } };
     // Set env variables for the hub to pull from. Add the startingBlockNumber and the hubConfig.
     setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber);
@@ -302,6 +296,63 @@ describe("ServerlessHub.js", function () {
 
     timeoutSpokeInstance.close();
   });
+  it("ServerlessHub does not page when every failing bot sets hubPageOnFailure: false", async function () {
+    const testBucket = "test-bucket"; // name of the config bucket.
+    const testConfigFile = "test-config-file"; // name of the config file.
+    const startingBlockNumber = Number(await provider.getBlockNumber());
+
+    const hubConfig = {
+      testServerlessMonitor: {
+        serverlessCommand: "true",
+        hubPageOnFailure: false,
+        environmentVariables: { CUSTOM_NODE_URL: network.config.url },
+      },
+    };
+    setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber);
+    setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig));
+
+    const testHubPort = 8085; // create a separate port to run this specific test on.
+    // Point the hub at a port nothing is listening on to force the spoke call to reject.
+    await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url);
+
+    const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort);
+
+    // The failure is still reported in full, just below the level the PagerDuty transport accepts.
+    assert.equal(lastSpyLogLevel(hubSpy), "warn");
+    assert.equal(rejectedResponse.res.statusCode, 500);
+    assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors"));
+    assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitor"));
+  });
+  it("ServerlessHub still pages when a bot without hubPageOnFailure: false fails alongside one with it", async function () {
+    const testBucket = "test-bucket"; // name of the config bucket.
+    const testConfigFile = "test-config-file"; // name of the config file.
+    const startingBlockNumber = Number(await provider.getBlockNumber());
+
+    const hubConfig = {
+      testServerlessMonitorNoPage: {
+        serverlessCommand: "true",
+        hubPageOnFailure: false,
+        environmentVariables: { CUSTOM_NODE_URL: network.config.url },
+      },
+      testServerlessMonitorPages: {
+        serverlessCommand: "true",
+        environmentVariables: { CUSTOM_NODE_URL: network.config.url },
+      },
+    };
+    setEnvironmentVariable(`lastQueriedBlockNumber-${defaultChainId}-${testConfigFile}`, startingBlockNumber);
+    setEnvironmentVariable(`${testBucket}-${testConfigFile}`, JSON.stringify(hubConfig));
+
+    const testHubPort = 8086; // create a separate port to run this specific test on.
+    // Point the hub at a port nothing is listening on to force both spoke calls to reject.
+    await hub.Poll(hubSpyLogger, testHubPort, "http://localhost:11111", network.config.url);
+
+    const rejectedResponse = await sendHubRequest({ bucket: testBucket, configFile: testConfigFile }, testHubPort);
+
+    assert.equal(lastSpyLogLevel(hubSpy), "error");
+    assert.equal(rejectedResponse.res.statusCode, 500);
+    assert.isTrue(lastSpyLogIncludes(hubSpy, "Some spoke calls returned errors"));
+    assert.isTrue(lastSpyLogIncludes(hubSpy, "testServerlessMonitorPages"));
+  });
   it("ServerlessHub can correctly execute multiple bots in parallel", async function () {
     // Set up the environment for testing. For these tests the hub is tested in `localStorage` mode where it will
     // read in hub configs and previous block numbers from the local storage of machine. This execution mode would be
@@ -593,10 +644,7 @@ describe("ServerlessHub.js", function () {
 
     // Logs should include correct starting and latest block numbers for the alternate network.
     const alternateBlockNumbers = {
-      [alternateChainId]: {
-        lastQueriedBlockNumber,
-        latestBlockNumber: latestAlternateBlockNumber,
-      },
+      [alternateChainId]: { lastQueriedBlockNumber, latestBlockNumber: latestAlternateBlockNumber },
     };
 
     // Strip enclosing curly braces as there are also other items in the logged object.