diff --git a/bin/commands/runs.js b/bin/commands/runs.js index de99a0a2..daea68b7 100644 --- a/bin/commands/runs.js +++ b/bin/commands/runs.js @@ -212,7 +212,7 @@ module.exports = function run(args, rawArgs) { logger.debug("Completed setting the configs"); if(!isBrowserstackInfra) { - if(process.env.BS_TESTOPS_BUILD_COMPLETED) { + if(process.env.BS_TESTOPS_BUILD_COMPLETED === "true") { setEventListeners(bsConfig); } @@ -226,7 +226,7 @@ module.exports = function run(args, rawArgs) { if(process.env.BROWSERSTACK_TEST_ACCESSIBILITY === 'true') { setAccessibilityEventListeners(bsConfig); } - if(process.env.BS_TESTOPS_BUILD_COMPLETED) { + if(process.env.BS_TESTOPS_BUILD_COMPLETED === "true") { setEventListeners(bsConfig); } markBlockEnd('validateConfig'); diff --git a/bin/helpers/capabilityHelper.js b/bin/helpers/capabilityHelper.js index f380b981..7f8bef59 100644 --- a/bin/helpers/capabilityHelper.js +++ b/bin/helpers/capabilityHelper.js @@ -4,7 +4,8 @@ const { readCypressConfigFile } = require('./readCypressConfigUtil'); const logger = require("./logger").winstonLogger, Constants = require("./constants"), - Utils = require("./utils"); + Utils = require("./utils"), + testhubUtils = require("../testhub/utils"); const caps = (bsConfig, zip) => { return new Promise(function (resolve, reject) { @@ -131,6 +132,18 @@ const caps = (bsConfig, zip) => { obj.run_settings = JSON.stringify(bsConfig.run_settings); } + // The only route by which a cypress session can name its TestHub build: every session + // this build spawns inherits these caps. Written unconditionally so an empty uuid records + // that build start ran and had nothing to name, which an absent key cannot express. + // "null" is the sentinel a failed build start leaves behind, not a uuid. + const testhubBuildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; + obj.testhubBuildUuid = Utils.isUndefined(testhubBuildUuid) || testhubBuildUuid === "null" + ? "" + : testhubBuildUuid; + obj.buildProductMap = testhubUtils.getProductMap(bsConfig); + + logger.debug(`TestHub build uuid stamped on caps: ${obj.testhubBuildUuid || ""}`); + obj.cypress_cli_user_agent = Utils.getUserAgent(); logger.info(`Cypress CLI User Agent: ${obj.cypress_cli_user_agent}`); diff --git a/bin/testhub/testhubHandler.js b/bin/testhub/testhubHandler.js index a6b4724a..e9cd1784 100644 --- a/bin/testhub/testhubHandler.js +++ b/bin/testhub/testhubHandler.js @@ -41,9 +41,9 @@ class TestHubHandler { const response = await nodeRequest( "POST", TESTHUB_CONSTANTS.TESTHUB_BUILD_API, data, config); const launchData = this.extractDataFromResponse(user_config, data, response, config); } catch (error) { - console.log(error); + logger.debug(`EXCEPTION IN BUILD START EVENT : ${error}`); + testhubUtils.handleErrorForObservability(error.success === false ? error : null); if (error.success === false) { // non 200 response - testhubUtils.logBuildError(error); return; } diff --git a/bin/testhub/utils.js b/bin/testhub/utils.js index 718bb595..aaca9b68 100644 --- a/bin/testhub/utils.js +++ b/bin/testhub/utils.js @@ -88,6 +88,10 @@ exports.setTestObservabilityVariables = ( }; exports.handleErrorForObservability = (error = null) => { + // Downstream reads isTestObservabilitySession(), not these ids, to decide whether + // observability is live. extractDataFromResponse clears it inline for a 2xx carrying + // success=false; this covers the paths that never get a usable response at all. + process.env.BROWSERSTACK_TEST_OBSERVABILITY = "false"; process.env.BROWSERSTACK_TESTHUB_UUID = "null"; process.env.BROWSERSTACK_TESTHUB_JWT = "null"; process.env.BS_TESTOPS_BUILD_COMPLETED = "false"; @@ -164,7 +168,7 @@ exports.handleErrorForAccessibility = (user_config, error = null) => { }; exports.logBuildError = (error, product = "") => { - if (error === undefined) { + if (isUndefined(error)) { logger.error(`${product.toUpperCase()} Build creation failed`); return; diff --git a/test/unit/bin/helpers/capabilityHelper.js b/test/unit/bin/helpers/capabilityHelper.js index 490d3dee..2cfc5699 100644 --- a/test/unit/bin/helpers/capabilityHelper.js +++ b/test/unit/bin/helpers/capabilityHelper.js @@ -5,6 +5,8 @@ const chai = require("chai"), const capabilityHelper = require("../../../../bin/helpers/capabilityHelper"), Constants = require("../../../../bin/helpers/constants"), + testhubUtils = require("../../../../bin/testhub/utils"), + o11yHelper = require("../../../../bin/testObservability/helper/helper"), logger = require("../../../../bin/helpers/logger").winstonLogger; chai.use(chaiAsPromised); @@ -562,6 +564,151 @@ describe("capabilityHelper.js", () => { }); }); }); + + context("testhub build attribution", () => { + const bsConfig = { + auth: { + username: "random", + access_key: "random", + }, + browsers: [ + { + browser: "chrome", + os: "Windows 10", + versions: ["78"], + }, + ], + run_settings: {}, + }; + const productMap = { + observability: true, + accessibility: false, + percy: false, + automate: true, + app_automate: false, + }; + let productMapStub; + let originalTesthubUuid; + + beforeEach(() => { + originalTesthubUuid = process.env.BROWSERSTACK_TESTHUB_UUID; + productMapStub = sinon.stub(testhubUtils, "getProductMap").returns(productMap); + }); + + afterEach(() => { + productMapStub.restore(); + if (originalTesthubUuid === undefined) { + delete process.env.BROWSERSTACK_TESTHUB_UUID; + } else { + process.env.BROWSERSTACK_TESTHUB_UUID = originalTesthubUuid; + } + }); + + it("stamps the testhub build uuid and the product map on the caps", () => { + process.env.BROWSERSTACK_TESTHUB_UUID = "some-testhub-build-uuid"; + return capabilityHelper + .caps(bsConfig, { zip_url: "bs://" }) + .then(function (data) { + let parsed_data = JSON.parse(data); + chai.assert.equal(parsed_data.testhubBuildUuid, "some-testhub-build-uuid"); + chai.assert.deepEqual(parsed_data.buildProductMap, productMap); + sinon.assert.calledWith(productMapStub, bsConfig); + }); + }); + + it("stamps an empty testhub build uuid when build start produced none", () => { + delete process.env.BROWSERSTACK_TESTHUB_UUID; + return capabilityHelper + .caps(bsConfig, { zip_url: "bs://" }) + .then(function (data) { + let parsed_data = JSON.parse(data); + chai.assert.equal(parsed_data.testhubBuildUuid, ""); + chai.assert.isTrue(Object.prototype.hasOwnProperty.call(parsed_data, "testhubBuildUuid")); + chai.assert.deepEqual(parsed_data.buildProductMap, productMap); + }); + }); + }); + + // These exercise the REAL getProductMap rather than a stub, because the thing under test is + // what the map SAYS, not that it is attached. + context("testhub build attribution — product map reflects reality", () => { + const ENV = ["BROWSERSTACK_TEST_OBSERVABILITY", "BROWSERSTACK_TESTHUB_UUID", + "BROWSERSTACK_TEST_ACCESSIBILITY", "BROWSERSTACK_AUTOMATION"]; + let saved; + + const bsConfigFor = (testObservability) => ({ + auth: { username: "random", access_key: "random" }, + browsers: [{ browser: "chrome", os: "Windows 10", versions: ["78"] }], + run_settings: { cypress_config_file: "./cypress.config.js" }, + testObservability, + }); + + beforeEach(() => { + saved = {}; + ENV.forEach((k) => { saved[k] = process.env[k]; }); + delete process.env.BROWSERSTACK_TEST_OBSERVABILITY; + process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false"; + process.env.BROWSERSTACK_AUTOMATION = "true"; + }); + + afterEach(() => { + ENV.forEach((k) => { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + }); + }); + + it("carries observability:false when the user explicitly disabled it in config", () => { + const bsConfig = bsConfigFor(false); + o11yHelper.setTestObservabilityFlags(bsConfig); + chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "false", "precondition"); + + return capabilityHelper + .caps(bsConfig, { zip_url: "bs://" }) + .then(function (data) { + const parsed_data = JSON.parse(data); + chai.assert.isFalse(parsed_data.buildProductMap.observability); + chai.assert.equal(parsed_data.testhubBuildUuid, ""); + }); + }); + + it("carries observability:true when the user asked for it and build start succeeded", () => { + const bsConfig = bsConfigFor(true); + o11yHelper.setTestObservabilityFlags(bsConfig); + process.env.BROWSERSTACK_TESTHUB_UUID = "a-real-build-uuid"; + + return capabilityHelper + .caps(bsConfig, { zip_url: "bs://" }) + .then(function (data) { + const parsed_data = JSON.parse(data); + chai.assert.isTrue(parsed_data.buildProductMap.observability); + chai.assert.equal(parsed_data.testhubBuildUuid, "a-real-build-uuid"); + }); + }); + + it("flips observability to false and drops the null sentinel when build start failed", () => { + const bsConfig = bsConfigFor(true); + o11yHelper.setTestObservabilityFlags(bsConfig); + chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "true", "precondition"); + + const errorStub = sinon.stub(logger, "error"); + try { + testhubUtils.handleErrorForObservability(); + } finally { + errorStub.restore(); + } + chai.assert.equal(process.env.BROWSERSTACK_TESTHUB_UUID, "null", "sentinel is what we guard against"); + + return capabilityHelper + .caps(bsConfig, { zip_url: "bs://" }) + .then(function (data) { + const parsed_data = JSON.parse(data); + chai.assert.isFalse(parsed_data.buildProductMap.observability); + chai.assert.equal(parsed_data.testhubBuildUuid, "", + 'the "null" sentinel must never be stamped as a uuid'); + }); + }); + }); }); describe("addCypressZipStartLocation", () => { diff --git a/test/unit/bin/testhub/utils.js b/test/unit/bin/testhub/utils.js new file mode 100644 index 00000000..5cc7eeee --- /dev/null +++ b/test/unit/bin/testhub/utils.js @@ -0,0 +1,111 @@ +const chai = require("chai"), + sinon = require("sinon"); + +const testhubUtils = require("../../../../bin/testhub/utils"), + logger = require("../../../../bin/helpers/logger").winstonLogger; + +describe("testhub/utils.js", () => { + const OBSERVABILITY_ENV = [ + "BROWSERSTACK_TEST_OBSERVABILITY", + "BROWSERSTACK_TESTHUB_UUID", + "BROWSERSTACK_TESTHUB_JWT", + "BS_TESTOPS_BUILD_COMPLETED", + "BS_TESTOPS_JWT", + "BS_TESTOPS_BUILD_HASHED_ID", + "BS_TESTOPS_ALLOW_SCREENSHOTS", + "BROWSERSTACK_TEST_ACCESSIBILITY", + "BROWSERSTACK_AUTOMATION", + ]; + let saved; + + beforeEach(() => { + saved = {}; + OBSERVABILITY_ENV.forEach((k) => { saved[k] = process.env[k]; }); + }); + + afterEach(() => { + OBSERVABILITY_ENV.forEach((k) => { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + }); + }); + + describe("getProductMap", () => { + it("reports observability false when the flag is not set to true", () => { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = "false"; + process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false"; + process.env.BROWSERSTACK_AUTOMATION = "true"; + + chai.assert.deepEqual(testhubUtils.getProductMap({}), { + observability: false, + accessibility: false, + percy: false, + automate: true, + app_automate: false, + }); + }); + + it("reports observability true only while the flag says so", () => { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true"; + process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false"; + process.env.BROWSERSTACK_AUTOMATION = "true"; + + chai.assert.isTrue(testhubUtils.getProductMap({}).observability); + }); + }); + + describe("handleErrorForObservability", () => { + let errorStub; + + beforeEach(() => { errorStub = sinon.stub(logger, "error"); }); + afterEach(() => { errorStub.restore(); }); + + it("turns observability off in the product map when build start fails", () => { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true"; + process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false"; + process.env.BROWSERSTACK_AUTOMATION = "true"; + chai.assert.isTrue(testhubUtils.getProductMap({}).observability, "precondition"); + + testhubUtils.handleErrorForObservability(); + + chai.assert.equal(process.env.BROWSERSTACK_TEST_OBSERVABILITY, "false"); + chai.assert.isFalse(testhubUtils.getProductMap({}).observability); + }); + + it("leaves the uuid as the null sentinel and marks the build not completed", () => { + process.env.BROWSERSTACK_TESTHUB_UUID = "some-uuid"; + process.env.BS_TESTOPS_BUILD_COMPLETED = "true"; + + testhubUtils.handleErrorForObservability(); + + chai.assert.equal(process.env.BROWSERSTACK_TESTHUB_UUID, "null"); + chai.assert.equal(process.env.BS_TESTOPS_BUILD_COMPLETED, "false"); + }); + + it("does not report observability as still enabled to shouldProcessEventForTesthub", () => { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = "true"; + process.env.BROWSERSTACK_TEST_ACCESSIBILITY = "false"; + + testhubUtils.handleErrorForObservability(); + + chai.assert.isFalse(testhubUtils.shouldProcessEventForTesthub()); + }); + }); + + describe("logBuildError", () => { + let errorStub; + + beforeEach(() => { errorStub = sinon.stub(logger, "error"); }); + afterEach(() => { errorStub.restore(); }); + + it("logs a readable message when there is no error object at all", () => { + testhubUtils.logBuildError(undefined, "observability"); + sinon.assert.calledWith(errorStub, "OBSERVABILITY Build creation failed"); + }); + + it("treats a null error the same as a missing one", () => { + testhubUtils.logBuildError(null, "observability"); + sinon.assert.calledWith(errorStub, "OBSERVABILITY Build creation failed"); + }); + }); +});