feat(o11y): stamp testhubBuildUuid + buildProductMap on cypress build caps [SDK-7285]

bin/commands/runs.js

@@ -212,7 +212,7 @@ module.exports = function run(args, rawArgs) {
     logger.debug("Completed setting the configs");
 
     if(!isBrowserstackInfra) {
-      if(process.env.BS_TESTOPS_BUILD_COMPLETED) {
+      if(process.env.BS_TESTOPS_BUILD_COMPLETED === "true") {
         setEventListeners(bsConfig);
       }
 
@@ -226,7 +226,7 @@ module.exports = function run(args, rawArgs) {
       if(process.env.BROWSERSTACK_TEST_ACCESSIBILITY === 'true') {
         setAccessibilityEventListeners(bsConfig);
       }
-      if(process.env.BS_TESTOPS_BUILD_COMPLETED) {
+      if(process.env.BS_TESTOPS_BUILD_COMPLETED === "true") {
         setEventListeners(bsConfig);
       }
       markBlockEnd('validateConfig');

bin/helpers/capabilityHelper.js

@@ -4,7 +4,8 @@ const { readCypressConfigFile } = require('./readCypressConfigUtil');
 
 const logger = require("./logger").winstonLogger,
   Constants = require("./constants"),
-  Utils = require("./utils");
+  Utils = require("./utils"),
+  testhubUtils = require("../testhub/utils");
 
 const caps = (bsConfig, zip) => {
   return new Promise(function (resolve, reject) {
@@ -131,6 +132,18 @@ const caps = (bsConfig, zip) => {
       obj.run_settings = JSON.stringify(bsConfig.run_settings);
     }
 
+    // The only route by which a cypress session can name its TestHub build: every session
+    // this build spawns inherits these caps. Written unconditionally so an empty uuid records
+    // that build start ran and had nothing to name, which an absent key cannot express.
+    // "null" is the sentinel a failed build start leaves behind, not a uuid.
+    const testhubBuildUuid = process.env.BROWSERSTACK_TESTHUB_UUID;
+    obj.testhubBuildUuid = Utils.isUndefined(testhubBuildUuid) || testhubBuildUuid === "null"
+      ? ""
+      : testhubBuildUuid;
+    obj.buildProductMap = testhubUtils.getProductMap(bsConfig);
+
+    logger.debug(`TestHub build uuid stamped on caps: ${obj.testhubBuildUuid || "<empty>"}`);
+
     obj.cypress_cli_user_agent = Utils.getUserAgent();
 
     logger.info(`Cypress CLI User Agent: ${obj.cypress_cli_user_agent}`);

bin/testhub/testhubHandler.js

@@ -41,9 +41,9 @@ class TestHubHandler {
       const response = await nodeRequest( "POST", TESTHUB_CONSTANTS.TESTHUB_BUILD_API, data, config);
       const launchData = this.extractDataFromResponse(user_config, data, response, config);
     } catch (error) {
-        console.log(error);
+        logger.debug(`EXCEPTION IN BUILD START EVENT : ${error}`);
+        testhubUtils.handleErrorForObservability(error.success === false ? error : null);
         if (error.success === false) { // non 200 response
-            testhubUtils.logBuildError(error);
             return;
         }
 

bin/testhub/utils.js

@@ -88,6 +88,10 @@ exports.setTestObservabilityVariables = (
 };
 
 exports.handleErrorForObservability = (error = null) => {
+  // Downstream reads isTestObservabilitySession(), not these ids, to decide whether
+  // observability is live. extractDataFromResponse clears it inline for a 2xx carrying
+  // success=false; this covers the paths that never get a usable response at all.
+  process.env.BROWSERSTACK_TEST_OBSERVABILITY = "false";
   process.env.BROWSERSTACK_TESTHUB_UUID = "null";
   process.env.BROWSERSTACK_TESTHUB_JWT = "null";
   process.env.BS_TESTOPS_BUILD_COMPLETED = "false";
@@ -164,7 +168,7 @@ exports.handleErrorForAccessibility = (user_config, error = null) => {
 };
 
 exports.logBuildError = (error, product = "") => {
-  if (error === undefined) {
+  if (isUndefined(error)) {
     logger.error(`${product.toUpperCase()} Build creation failed`);
 
     return;

test/unit/bin/helpers/capabilityHelper.js

@@ -5,6 +5,8 @@ const chai = require("chai"),
 
 const capabilityHelper = require("../../../../bin/helpers/capabilityHelper"),
   Constants = require("../../../../bin/helpers/constants"),
+  testhubUtils = require("../../../../bin/testhub/utils"),
+  o11yHelper = require("../../../../bin/testObservability/helper/helper"),
   logger = require("../../../../bin/helpers/logger").winstonLogger;
 
 chai.use(chaiAsPromised);
@@ -562,6 +564,151 @@ describe("capabilityHelper.js", () => {
           });
       });
     });
+
+    context("testhub build attribution", () => {
+      const bsConfig = {
+        auth: {
+          username: "random",
+          access_key: "random",
+        },
+        browsers: [
+          {
+            browser: "chrome",
+            os: "Windows 10",
+            versions: ["78"],
+          },
+        ],
+        run_settings: {},
+      };
+      const productMap = {
+        observability: true,
+        accessibility: false,
+        percy: false,
+        automate: true,
+        app_automate: false,
+      };
+      let productMapStub;
+      let originalTesthubUuid;
+
+      beforeEach(() => {
+        originalTesthubUuid = process.env.BROWSERSTACK_TESTHUB_UUID;
+        productMapStub = sinon.stub(testhubUtils, "getProductMap").returns(productMap);
+      });
+
+      afterEach(() => {
+        productMapStub.restore();
+        if (originalTesthubUuid === undefined) {
+          delete process.env.BROWSERSTACK_TESTHUB_UUID;
+        } else {
+          process.env.BROWSERSTACK_TESTHUB_UUID = originalTesthubUuid;
+        }
+      });
+
+      it("stamps the testhub build uuid and the product map on the caps", () => {
+        process.env.BROWSERSTACK_TESTHUB_UUID = "some-testhub-build-uuid";
+        return capabilityHelper
+          .caps(bsConfig, { zip_url: "bs://<random>" })
+          .then(function (data) {
+            let parsed_data = JSON.parse(data);
+            chai.assert.equal(parsed_data.testhubBuildUuid, "some-testhub-build-uuid");
+            chai.assert.deepEqual(parsed_data.buildProductMap, productMap);
+            sinon.assert.calledWith(productMapStub, bsConfig);
+          });
+      });
+
+      it("stamps an empty testhub build uuid when build start produced none", () => {
+        delete process.env.BROWSERSTACK_TESTHUB_UUID;
+        return capabilityHelper
+          .caps(bsConfig, { zip_url: "bs://<random>" })
+          .then(function (data) {
+            let parsed_data = JSON.parse(data);
+            chai.assert.equal(parsed_data.testhubBuildUuid, "");
+            chai.assert.isTrue(Object.prototype.hasOwnProperty.call(parsed_data, "testhubBuildUuid"));
+            chai.assert.deepEqual(parsed_data.buildProductMap, productMap);
+          });
+      });
+    });
+
+    // These exercise the REAL getProductMap rather than a stub, because the thing under test is
+    // what the map SAYS, not that it is attached.
+    context("testhub build attribution — product map reflects reality", () => {
+      const ENV = ["BROWSERSTACK_TEST_OBSERVABILITY", "BROWSERSTACK_TESTHUB_UUID",
+                   "BROWSERSTACK_TEST_ACCESSIBILITY", "BROWSERSTACK_AUTOMATION"];
+      let saved;
+
+      const bsConfigFor = (testObservability) => ({
+        auth: { username: "random", access_key: "random" },
+        browsers: [{ browser: "chrome", os: "Windows 10", versions: ["78"] }],
+        run_settings: { cypress_config_file: "./cypress.config.js" },
+        testObservability,
+      });
+
+      beforeEach(() => {
+        saved = {};
+        ENV.forEach((k) => { saved[k] = process.env[k]; });
+        delete process.env.BROWSERSTACK_TEST_OBSERVABILITY;
+        process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false";
+        process.env.BROWSERSTACK_AUTOMATION = "true";
+      });
+
+      afterEach(() => {
+        ENV.forEach((k) => {
+          if (saved[k] === undefined) delete process.env[k];
+          else process.env[k] = saved[k];
+        });
+      });
+
+      it("carries observability:false when the user explicitly disabled it in config", () => {
+        const bsConfig = bsConfigFor(false);
+        o11yHelper.setTestObservabilityFlags(bsConfig);
+        chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "false", "precondition");
+
+        return capabilityHelper
+          .caps(bsConfig, { zip_url: "bs://<random>" })
+          .then(function (data) {
+            const parsed_data = JSON.parse(data);
+            chai.assert.isFalse(parsed_data.buildProductMap.observability);
+            chai.assert.equal(parsed_data.testhubBuildUuid, "");
+          });
+      });
+
+      it("carries observability:true when the user asked for it and build start succeeded", () => {
+        const bsConfig = bsConfigFor(true);
+        o11yHelper.setTestObservabilityFlags(bsConfig);
+        process.env.BROWSERSTACK_TESTHUB_UUID = "a-real-build-uuid";
+
+        return capabilityHelper
+          .caps(bsConfig, { zip_url: "bs://<random>" })
+          .then(function (data) {
+            const parsed_data = JSON.parse(data);
+            chai.assert.isTrue(parsed_data.buildProductMap.observability);
+            chai.assert.equal(parsed_data.testhubBuildUuid, "a-real-build-uuid");
+          });
+      });
+
+      it("flips observability to false and drops the null sentinel when build start failed", () => {
+        const bsConfig = bsConfigFor(true);
+        o11yHelper.setTestObservabilityFlags(bsConfig);
+        chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "true", "precondition");
+
+        const errorStub = sinon.stub(logger, "error");
+        try {
+          testhubUtils.handleErrorForObservability();
+        } finally {
+          errorStub.restore();
+        }
+        chai.assert.equal(process.env.BROWSERSTACK_TESTHUB_UUID, "null", "sentinel is what we guard against");
+
+        return capabilityHelper
+          .caps(bsConfig, { zip_url: "bs://<random>" })
+          .then(function (data) {
+            const parsed_data = JSON.parse(data);
+            chai.assert.isFalse(parsed_data.buildProductMap.observability);
+            chai.assert.equal(parsed_data.testhubBuildUuid, "",
+              'the "null" sentinel must never be stamped as a uuid');
+          });
+      });
+    });
   });
 
   describe("addCypressZipStartLocation", () => {

test/unit/bin/testhub/utils.js

@@ -0,0 +1,111 @@
+const chai = require("chai"),
+  sinon = require("sinon");
+
+const testhubUtils = require("../../../../bin/testhub/utils"),
+  logger = require("../../../../bin/helpers/logger").winstonLogger;
+
+describe("testhub/utils.js", () => {
+  const OBSERVABILITY_ENV = [
+    "BROWSERSTACK_TEST_OBSERVABILITY",
+    "BROWSERSTACK_TESTHUB_UUID",
+    "BROWSERSTACK_TESTHUB_JWT",
+    "BS_TESTOPS_BUILD_COMPLETED",
+    "BS_TESTOPS_JWT",
+    "BS_TESTOPS_BUILD_HASHED_ID",
+    "BS_TESTOPS_ALLOW_SCREENSHOTS",
+    "BROWSERSTACK_TEST_ACCESSIBILITY",
+    "BROWSERSTACK_AUTOMATION",
+  ];
+  let saved;
+
+  beforeEach(() => {
+    saved = {};
+    OBSERVABILITY_ENV.forEach((k) => { saved[k] = process.env[k]; });
+  });
+
+  afterEach(() => {
+    OBSERVABILITY_ENV.forEach((k) => {
+      if (saved[k] === undefined) delete process.env[k];
+      else process.env[k] = saved[k];
+    });
+  });
+
+  describe("getProductMap", () => {
+    it("reports observability false when the flag is not set to true", () => {
+      process.env.BROWSERSTACK_TEST_OBSERVABILITY = "false";
+      process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false";
+      process.env.BROWSERSTACK_AUTOMATION = "true";
+
+      chai.assert.deepEqual(testhubUtils.getProductMap({}), {
+        observability: false,
+        accessibility: false,
+        percy: false,
+        automate: true,
+        app_automate: false,
+      });
+    });
+
+    it("reports observability true only while the flag says so", () => {
+      process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true";
+      process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false";
+      process.env.BROWSERSTACK_AUTOMATION = "true";
+
+      chai.assert.isTrue(testhubUtils.getProductMap({}).observability);
+    });
+  });
+
+  describe("handleErrorForObservability", () => {
+    let errorStub;
+
+    beforeEach(() => { errorStub = sinon.stub(logger, "error"); });
+    afterEach(() => { errorStub.restore(); });
+
+    it("turns observability off in the product map when build start fails", () => {
+      process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true";
+      process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false";
+      process.env.BROWSERSTACK_AUTOMATION = "true";
+      chai.assert.isTrue(testhubUtils.getProductMap({}).observability, "precondition");
+
+      testhubUtils.handleErrorForObservability();
+
+      chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "false");
+      chai.assert.isFalse(testhubUtils.getProductMap({}).observability);
+    });
+
+    it("leaves the uuid as the null sentinel and marks the build not completed", () => {
+      process.env.BROWSERSTACK_TESTHUB_UUID = "some-uuid";
+      process.env.BS_TESTOPS_BUILD_COMPLETED = "true";
+
+      testhubUtils.handleErrorForObservability();
+
+      chai.assert.equal(process.env.BROWSERSTACK_TESTHUB_UUID, "null");
+      chai.assert.equal(process.env.BS_TESTOPS_BUILD_COMPLETED, "false");
+    });
+
+    it("does not report observability as still enabled to shouldProcessEventForTesthub", () => {
+      process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true";
+      process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false";
+
+      testhubUtils.handleErrorForObservability();
+
+      chai.assert.isFalse(testhubUtils.shouldProcessEventForTesthub());
+    });
+  });
+
+  describe("logBuildError", () => {
+    let errorStub;
+
+    beforeEach(() => { errorStub = sinon.stub(logger, "error"); });
+    afterEach(() => { errorStub.restore(); });
+
+    it("logs a readable message when there is no error object at all", () => {
+      testhubUtils.logBuildError(undefined, "observability");
+      sinon.assert.calledWith(errorStub, "OBSERVABILITY Build creation failed");
+    });
+
+    it("treats a null error the same as a missing one", () => {
+      testhubUtils.logBuildError(null, "observability");
+      sinon.assert.calledWith(errorStub, "OBSERVABILITY Build creation failed");
+    });
+  });
+});