diff --git a/packages/kernel-test/src/cluster-launch.test.ts b/packages/kernel-test/src/cluster-launch.test.ts index bc28f7224..4b7e884a7 100644 --- a/packages/kernel-test/src/cluster-launch.test.ts +++ b/packages/kernel-test/src/cluster-launch.test.ts @@ -1,4 +1,5 @@ import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; +import { waitUntilQuiescent } from '@metamask/kernel-utils'; import { Logger } from '@metamask/logger'; import type { LogEntry } from '@metamask/logger'; import type { Kernel } from '@metamask/ocap-kernel'; @@ -123,3 +124,50 @@ describe('cluster initialization', { timeout: 4_000 }, () => { ]); }); }); + +describe('peer rejection propagation', { timeout: 10_000 }, () => { + let logger: Logger; + let entries: LogEntry[]; + let kernel: Kernel; + + beforeEach(async () => { + const testLogger = makeTestLogger(); + logger = testLogger.logger; + entries = testLogger.entries; + const database = await makeSQLKernelDatabase({}); + kernel = await makeKernel( + database, + true, + logger.subLogger({ tags: ['test'] }), + ); + }); + + it('bootstrap observes peer rejection when a peer vat fails to launch', async () => { + await expect( + kernel.launchSubcluster({ + bootstrap: 'main', + vats: { + main: { + bundleSpec: getBundleSpec('peer-rejection-bootstrap'), + parameters: {}, + }, + peer: { + bundleSpec: getBundleSpec('error-build-throw'), + parameters: {}, + }, + }, + }), + ).rejects.toMatchObject({ + message: expect.stringMatching(/^Failed to launch vat \S+ \(peer\)$/u), + }); + + // Let the kernel run loop deliver the parked bootstrap message to the + // main vat, which will observe the peer's rejected root promise. + await waitUntilQuiescent(200); + + const vatLogs = extractTestLogs(entries, 'console'); + expect(vatLogs).toContainEqual( + expect.stringMatching(/^peer rejected:.*VAT_TERMINATED/u), + ); + }); +}); diff --git a/packages/kernel-test/src/persistence.test.ts b/packages/kernel-test/src/persistence.test.ts index c4dd6ec81..e8b6e2781 100644 --- a/packages/kernel-test/src/persistence.test.ts +++ b/packages/kernel-test/src/persistence.test.ts @@ -96,10 +96,15 @@ describe('persistent storage', { timeout: 20_000 }, () => { false, logger.logger.subLogger({ tags: ['test'] }), ); - const result1 = await runTestVats(kernel1, multiVatCluster); - expect(result1).toBe('Coordinator initialized with 2 workers'); + // Capture rootKref directly: concurrent vat launch means the coordinator + // may not be assigned ko4, so we cannot use a hardcoded ref here. + const { bootstrapResult: launch1Result, rootKref: coordinatorRoot } = + await kernel1.launchSubcluster(multiVatCluster); await waitUntilQuiescent(); - const workResult1 = await runResume(kernel1, v1Root); + expect(kunser(launch1Result as CapData)).toBe( + 'Coordinator initialized with 2 workers', + ); + const workResult1 = await runResume(kernel1, coordinatorRoot); expect(workResult1).toBe('Work completed: Worker1(1), Worker2(1)'); await waitUntilQuiescent(); await kernel1.stop(); @@ -110,7 +115,8 @@ describe('persistent storage', { timeout: 20_000 }, () => { logger.logger.subLogger({ tags: ['test'] }), ); await new Promise((resolve) => setTimeout(resolve, 1000)); - const workResult2 = await runResume(kernel2, v1Root); + // coordinatorRoot (ko) is stable across kernel restarts. + const workResult2 = await runResume(kernel2, coordinatorRoot); expect(workResult2).toBe('Work completed: Worker1(2), Worker2(2)'); await kernel2.stop(); }); diff --git a/packages/kernel-test/src/rejection.test.ts b/packages/kernel-test/src/rejection.test.ts index 3e6c8266e..ba268e484 100644 --- a/packages/kernel-test/src/rejection.test.ts +++ b/packages/kernel-test/src/rejection.test.ts @@ -31,7 +31,7 @@ describe('rejection', () => { }); expect(vat).toBeDefined(); const vats = kernel.getVatIds(); - expect(vats).toStrictEqual(vatIds); + expect([...vats].sort()).toStrictEqual([...vatIds].sort()); await waitUntilQuiescent(); const vatLogs = vatIds.map((vatId) => extractTestLogs(entries, vatId)); diff --git a/packages/kernel-test/src/resume.test.ts b/packages/kernel-test/src/resume.test.ts index 0f74aff9f..68808368f 100644 --- a/packages/kernel-test/src/resume.test.ts +++ b/packages/kernel-test/src/resume.test.ts @@ -1,6 +1,7 @@ +import type { CapData } from '@endo/marshal'; import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { waitUntilQuiescent } from '@metamask/kernel-utils'; -import type { KRef } from '@metamask/ocap-kernel'; +import { kunser } from '@metamask/ocap-kernel'; import { describe, expect, it } from 'vitest'; import { @@ -8,7 +9,6 @@ import { makeKernel, makeTestLogger, runResume, - runTestVats, sortLogs, extractTestLogs, } from './utils.ts'; @@ -100,11 +100,6 @@ const reference = sortLogs([ ...carolResumeReference, ]); -// Vat root objects start with ko4 due to the kernel facet and other kernel service objects being created before any vats. -const v1Root: KRef = 'ko4'; -const v2Root: KRef = 'ko5'; -const v3Root: KRef = 'ko6'; - describe('restarting vats', async () => { it('exercise restart vats individually', async () => { const kernelDatabase = await makeSQLKernelDatabase({ @@ -112,9 +107,15 @@ describe('restarting vats', async () => { }); const { logger, entries } = makeTestLogger(); const kernel = await makeKernel(kernelDatabase, true, logger); - const bootstrapResult = await runTestVats(kernel, testSubcluster); - expect(bootstrapResult).toBe('bootstrap Alice'); + // Use launchSubcluster directly to get vatRootKrefs: concurrent vat launch + // means ko assignment order depends on worker startup speed. + const { bootstrapResult, vatRootKrefs } = + await kernel.launchSubcluster(testSubcluster); await waitUntilQuiescent(); + expect(kunser(bootstrapResult as CapData)).toBe('bootstrap Alice'); + const v1Root = vatRootKrefs.alice; + const v2Root = vatRootKrefs.bob; + const v3Root = vatRootKrefs.carol; await kernel.restartVat('v1'); await kernel.restartVat('v2'); await kernel.restartVat('v3'); @@ -136,9 +137,15 @@ describe('restarting vats', async () => { }); const { logger: logger1, entries: entries1 } = makeTestLogger(); const kernel1 = await makeKernel(kernelDatabase, true, logger1); - const bootstrapResult = await runTestVats(kernel1, testSubcluster); - expect(bootstrapResult).toBe('bootstrap Alice'); + // Capture vatRootKrefs from first kernel: ko refs are stable across + // kernel restarts because they are persisted in the kernel store. + const { bootstrapResult, vatRootKrefs } = + await kernel1.launchSubcluster(testSubcluster); await waitUntilQuiescent(); + expect(kunser(bootstrapResult as CapData)).toBe('bootstrap Alice'); + const v1Root = vatRootKrefs.alice; + const v2Root = vatRootKrefs.bob; + const v3Root = vatRootKrefs.carol; const { logger: logger2, entries: entries2 } = makeTestLogger(); const kernel2 = await makeKernel(kernelDatabase, false, logger2); await new Promise((resolve) => setTimeout(resolve, 1000)); diff --git a/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts b/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts new file mode 100644 index 000000000..890102ef9 --- /dev/null +++ b/packages/kernel-test/src/vats/peer-rejection-bootstrap.ts @@ -0,0 +1,28 @@ +import { E } from '@endo/eventual-send'; +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Bootstrap vat for testing peer-rejection propagation. + * Receives a `peer` root reference that may be a rejected kernel promise + * (e.g. if the peer vat failed to launch), and logs whether calls resolve + * or reject so integration tests can inspect the outcome. + * + * @returns The root object for this vat. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function buildRootObject() { + return makeDefaultExo('root', { + async bootstrap({ peer }: { peer: unknown }) { + await E(peer as object) + .ping() + // eslint-disable-next-line no-console + .then(() => console.log('peer resolved')) + .catch((error: unknown) => { + const message = + error instanceof Error ? error.message : String(error); + // eslint-disable-next-line no-console + console.log(`peer rejected: ${message}`); + }); + }, + }); +} diff --git a/packages/ocap-kernel/CHANGELOG.md b/packages/ocap-kernel/CHANGELOG.md index de6c99c0e..a452df364 100644 --- a/packages/ocap-kernel/CHANGELOG.md +++ b/packages/ocap-kernel/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Launch all vats in a subcluster concurrently during `launchSubcluster`, reducing startup latency from serial to parallel; failed peer vats receive a rejected kernel promise observable via `E(roots.peer).method()` pipelining ([#983](https://github.com/MetaMask/ocap-kernel/pull/983)) + - Add `fetch`, `Request`, `Headers`, and `Response` to available vat endowments ([#942](https://github.com/MetaMask/ocap-kernel/pull/942)) - Add `VatConfig.network: { allowedHosts: string[] }`; requesting `'fetch'` without it rejects `initVat` - Integrate Snaps attenuated endowment factories into vat globals ([#937](https://github.com/MetaMask/ocap-kernel/pull/937)) diff --git a/packages/ocap-kernel/src/Kernel.test.ts b/packages/ocap-kernel/src/Kernel.test.ts index ca1dde78b..625b5378d 100644 --- a/packages/ocap-kernel/src/Kernel.test.ts +++ b/packages/ocap-kernel/src/Kernel.test.ts @@ -34,6 +34,8 @@ const mocks = vi.hoisted(() => { } waitForCrank = vi.fn().mockResolvedValue(undefined); + + resolvePromises = vi.fn(); } class RemoteManager { @@ -299,6 +301,7 @@ describe('Kernel', () => { subclusterId: 's1', bootstrapResult: { body: '{"result":"ok"}', slots: [] }, rootKref: expect.stringMatching(/^ko\d+$/u), + vatRootKrefs: { alice: expect.stringMatching(/^ko\d+$/u) }, }); }); }); diff --git a/packages/ocap-kernel/src/store/methods/vat.ts b/packages/ocap-kernel/src/store/methods/vat.ts index 84e065ee5..48d431dc1 100644 --- a/packages/ocap-kernel/src/store/methods/vat.ts +++ b/packages/ocap-kernel/src/store/methods/vat.ts @@ -44,7 +44,7 @@ export function getVatMethods(ctx: StoreContext) { getKernelPromise, addPromiseSubscriber, } = getPromiseMethods(ctx); - const { initKernelObject } = getObjectMethods(ctx); + const { initKernelObject, getObjectRefCount } = getObjectMethods(ctx); const { addCListEntry } = getCListMethods(ctx); const { incrementRefCount, decrementRefCount } = getRefCountMethods(ctx); @@ -261,8 +261,11 @@ export function getVatMethods(ctx: StoreContext) { const { vatSlot } = getReachableAndVatSlot(vatID, kref); ctx.kv.delete(getSlotKey(vatID, kref)); ctx.kv.delete(getSlotKey(vatID, vatSlot)); - // Decrease refcounts that belonged to the terminating vat - decrementRefCount(kref, 'cleanup|export|baseline'); + // Skip baseline decrement if GC already zeroed reachable via dropImports. + const { reachable } = getObjectRefCount(kref); + if (reachable > 0) { + decrementRefCount(kref, 'cleanup|export|baseline'); + } ctx.maybeFreeKrefs.add(kref); work.exports += 1; } diff --git a/packages/ocap-kernel/src/types.ts b/packages/ocap-kernel/src/types.ts index e5ea314b0..b265e4775 100644 --- a/packages/ocap-kernel/src/types.ts +++ b/packages/ocap-kernel/src/types.ts @@ -746,6 +746,8 @@ export type SubclusterLaunchResult = { rootKref: KRef; /** The CapData result of calling bootstrap() on the root object, if any. */ bootstrapResult: CapData | undefined; + /** Map from vat name to root kref for all successfully launched vats. */ + vatRootKrefs: Record; }; const RemoteCommsDisconnectedStruct = object({ diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts index 43c15657c..b8dd897d7 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.test.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.test.ts @@ -4,7 +4,7 @@ import type { Mocked } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { KernelQueue } from '../KernelQueue.ts'; -import { kser } from '../liveslots/kernel-marshal.ts'; +import { kser, makeKernelError } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { VatId, @@ -65,10 +65,15 @@ describe('SubclusterManager', () => { getRootObject: vi.fn(), deleteVatConfig: vi.fn(), markVatAsTerminated: vi.fn(), + initKernelPromise: vi + .fn() + .mockReturnValue(['kp1', { state: 'unresolved', subscribers: [] }]), + setPromiseDecider: vi.fn(), } as unknown as Mocked; mockKernelQueue = { waitForCrank: vi.fn().mockResolvedValue(undefined), + resolvePromises: vi.fn(), } as unknown as Mocked; mockVatManager = { @@ -76,6 +81,7 @@ describe('SubclusterManager', () => { terminateVat: vi.fn().mockResolvedValue(undefined), collectGarbage: vi.fn(), terminateAllVats: vi.fn().mockResolvedValue(undefined), + hasVat: vi.fn().mockReturnValue(false), } as unknown as Mocked; mockGetKernelService = vi.fn().mockReturnValue(undefined) as unknown as ( @@ -116,14 +122,17 @@ describe('SubclusterManager', () => { 'testVat', 's1', ); + // queueMessage targets the real root KRef once all vats have launched expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ { testVat: expect.anything() }, {}, ]); + expect(mockKernelQueue.resolvePromises).not.toHaveBeenCalled(); expect(result).toStrictEqual({ subclusterId: 's1', rootKref: 'ko1', bootstrapResult: { body: '{"result":"ok"}', slots: [] }, + vatRootKrefs: { testVat: 'ko1' }, }); }); @@ -152,6 +161,68 @@ describe('SubclusterManager', () => { 'bob', 's1', ); + // bootstrap receives real ko refs for both vats; no kernel promises allocated + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + { alice: expect.anything(), bob: expect.anything() }, + {}, + ]); + expect(mockKernelQueue.resolvePromises).not.toHaveBeenCalled(); + }); + + it('rejects peer kernel promise when a non-bootstrap vat fails to launch', async () => { + const config: ClusterConfig = { + bootstrap: 'alice', + vats: { + alice: { sourceSpec: 'alice.js' }, + bob: { sourceSpec: 'bob.js' }, + }, + }; + const bobError = new Error('bob exploded'); + mockVatManager.launchVat + .mockResolvedValueOnce('ko1' as KRef) + .mockRejectedValueOnce(bobError); + + await expect(subclusterManager.launchSubcluster(config)).rejects.toThrow( + 'bob exploded', + ); + + // bootstrap receives alice's real ko1 root ref as queueMessage target + expect(mockQueueMessage).toHaveBeenCalledWith('ko1', 'bootstrap', [ + { alice: expect.anything(), bob: expect.anything() }, + {}, + ]); + // initKernelPromise called once — only for bob's rejected promise + expect(mockKernelStore.initKernelPromise).toHaveBeenCalledTimes(1); + // bob's promise rejected with a VAT_TERMINATED kernel error + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledTimes(1); + expect(mockKernelQueue.resolvePromises).toHaveBeenCalledWith('kernel', [ + ['kp1', true, makeKernelError('VAT_TERMINATED', 'bob exploded')], + ]); + }); + + it('terminates successfully-launched vats when a peer vat fails', async () => { + const config: ClusterConfig = { + bootstrap: 'alice', + vats: { + alice: { sourceSpec: 'alice.js' }, + bob: { sourceSpec: 'bob.js' }, + }, + }; + mockVatManager.launchVat + .mockResolvedValueOnce('ko1' as KRef) + .mockRejectedValueOnce(new Error('bob exploded')); + // Simulate alice's vatId being registered in the subcluster store. + mockKernelStore.getSubclusterVats.mockReturnValue(['v1' as VatId]); + // alice's vat worker is running + (mockVatManager.hasVat as ReturnType).mockReturnValue(true); + + await expect(subclusterManager.launchSubcluster(config)).rejects.toThrow( + 'bob exploded', + ); + + expect(mockVatManager.terminateVat).toHaveBeenCalledWith('v1'); + expect(mockVatManager.collectGarbage).toHaveBeenCalled(); + expect(mockKernelStore.deleteSubcluster).toHaveBeenCalledWith('s1'); }); it('includes unrestricted kernel services when specified', async () => { @@ -303,6 +374,12 @@ describe('SubclusterManager', () => { }); mockVatManager.launchVat.mockRejectedValue(new Error('vat boom')); + // Service lookup now happens before vat launch, so the IO channel + // service must be registered for the test to reach the vat launch step. + (mockGetKernelService as ReturnType).mockReturnValue({ + kref: 'ko99', + systemOnly: false, + }); const config: ClusterConfig = { bootstrap: 'testVat', @@ -365,6 +442,12 @@ describe('SubclusterManager', () => { }); mockVatManager.launchVat.mockRejectedValue(new Error('launch boom')); + // Service lookup now happens before vat launch, so the IO channel + // service must be registered for the test to reach the vat launch step. + (mockGetKernelService as ReturnType).mockReturnValue({ + kref: 'ko99', + systemOnly: false, + }); const config: ClusterConfig = { bootstrap: 'testVat', @@ -430,6 +513,7 @@ describe('SubclusterManager', () => { subclusterId: 's1', rootKref: 'ko1', bootstrapResult, + vatRootKrefs: { testVat: 'ko1' }, }); }); diff --git a/packages/ocap-kernel/src/vats/SubclusterManager.ts b/packages/ocap-kernel/src/vats/SubclusterManager.ts index a32ac7a5d..505726414 100644 --- a/packages/ocap-kernel/src/vats/SubclusterManager.ts +++ b/packages/ocap-kernel/src/vats/SubclusterManager.ts @@ -5,7 +5,7 @@ import { Logger } from '@metamask/logger'; import type { IOManager } from '../io/IOManager.ts'; import type { KernelQueue } from '../KernelQueue.ts'; import type { VatManager } from './VatManager.ts'; -import { kslot, kunser } from '../liveslots/kernel-marshal.ts'; +import { kslot, kunser, makeKernelError } from '../liveslots/kernel-marshal.ts'; import type { SlotValue } from '../liveslots/kernel-marshal.ts'; import type { KernelStore } from '../store/index.ts'; import type { @@ -135,14 +135,28 @@ export class SubclusterManager { } this.#validateServices(config, isSystem); - const { rootKref, bootstrapResult } = await this.#launchVatsForSubcluster( - subclusterId, - config, - ); - return { subclusterId, rootKref, bootstrapResult }; + const { rootKref, bootstrapResult, vatRootKrefs } = + await this.#launchVatsForSubcluster(subclusterId, config); + return { subclusterId, rootKref, bootstrapResult, vatRootKrefs }; } catch (error) { - // Roll back IO channels and persisted subcluster on failure. - // Cleanup is best-effort — errors must not mask the original failure. + // Roll back: terminate any vats that launched successfully, then clean + // up IO channels and the persisted subcluster record. + // Each step is best-effort — cleanup errors must not mask the original + // failure. + try { + const vatIds = this.#kernelStore.getSubclusterVats(subclusterId); + for (const vatId of vatIds.reverse()) { + if (this.#vatManager.hasVat(vatId)) { + await this.#vatManager.terminateVat(vatId); + this.#vatManager.collectGarbage(); + } + } + } catch (vatCleanupError) { + this.#logger.error( + 'Error during vat cleanup on failed launch:', + vatCleanupError, + ); + } try { if (this.#ioManager) { await this.#ioManager.destroyChannels(subclusterId); @@ -301,18 +315,10 @@ export class SubclusterManager { ): Promise<{ rootKref: KRef; bootstrapResult: CapData | undefined; + vatRootKrefs: Record; }> { - const rootIds: Record = {}; - const roots: Record = {}; - for (const [vatName, vatConfig] of Object.entries(config.vats)) { - const rootRef = await this.#vatManager.launchVat( - vatConfig, - vatName, - subclusterId, - ); - rootIds[vatName] = rootRef; - roots[vatName] = kslot(rootRef, 'vatRoot'); - } + const vatEntries = Object.entries(config.vats); + const services: Record = {}; const ioNames = config.io ? new Set(Object.keys(config.io)) @@ -334,12 +340,67 @@ export class SubclusterManager { throw Error(`no registered kernel service '${lookupName}'`); } } - const rootKref = rootIds[config.bootstrap]; - if (!rootKref) { - throw new Error( - `Bootstrap vat "${config.bootstrap}" not found in rootIds`, - ); + + // Launch all vats concurrently. getNextVatId() runs synchronously before + // each launchVat's first await, so vat-ID allocation order is deterministic. + const launchResults = await Promise.allSettled( + vatEntries.map(async ([vatName, vatConfig]) => + this.#vatManager.launchVat(vatConfig, vatName, subclusterId), + ), + ); + + // If the bootstrap vat itself failed to launch, throw immediately. + const bootstrapIdx = vatEntries.findIndex( + ([name]) => name === config.bootstrap, + ); + // bootstrapIdx is guaranteed ≥ 0 because launchSubcluster validates that + // config.vats[config.bootstrap] exists. The undefined guard satisfies tsc. + const bootstrapLaunchResult = launchResults[bootstrapIdx]; + if (bootstrapLaunchResult === undefined) { + throw Fail`no launch result for bootstrap vat '${config.bootstrap}'`; + } + if (bootstrapLaunchResult.status === 'rejected') { + throw bootstrapLaunchResult.reason; + } + const rootKref = bootstrapLaunchResult.value; + + // Build the roots map. Succeeded vats receive real ko refs; failed peer + // vats receive an immediately-rejected kernel promise so bootstrap can + // observe the failure via E(roots.peer).method() pipelining. + const rootsEntries: [string, SlotValue][] = []; + const vatRootKrefsEntries: [string, KRef][] = []; + let firstPeerFailure: Error | undefined; + for (let i = 0; i < vatEntries.length; i++) { + const vatEntry = vatEntries[i]; + const result = launchResults[i]; + if (vatEntry === undefined || result === undefined) { + throw Fail`missing entry at index ${i}`; + } + const [vatName] = vatEntry; + // Reject vat names that shadow Object.prototype built-ins (__proto__, + // constructor, etc.) before using them as property keys. + !(vatName in Object.prototype) || Fail`invalid vat name '${vatName}'`; + if (result.status === 'fulfilled') { + rootsEntries.push([vatName, kslot(result.value, 'vatRoot')]); + vatRootKrefsEntries.push([vatName, result.value]); + } else { + // launchVat always wraps failures in new Error(...), so reason is an Error + const peerError = + result.reason instanceof Error + ? result.reason + : new Error(String(result.reason)); + firstPeerFailure ??= peerError; + const [kpid] = this.#kernelStore.initKernelPromise(); + this.#kernelStore.setPromiseDecider(kpid, 'kernel'); + this.#kernelQueue.resolvePromises('kernel', [ + [kpid, true, makeKernelError('VAT_TERMINATED', peerError.message)], + ]); + rootsEntries.push([vatName, kslot(kpid)]); + } } + const roots = Object.fromEntries(rootsEntries); + const vatRootKrefs = Object.fromEntries(vatRootKrefsEntries); + const bootstrapResult = await this.#queueMessage(rootKref, 'bootstrap', [ roots, services, @@ -348,7 +409,14 @@ export class SubclusterManager { if (unserialized instanceof Error) { throw unserialized; } - return { rootKref, bootstrapResult }; + + // If any peer vats failed to launch, propagate after bootstrap has had a + // chance to observe and handle the failures. + if (firstPeerFailure !== undefined) { + throw firstPeerFailure; + } + + return { rootKref, bootstrapResult, vatRootKrefs }; } /**