metro-file-map: Lazy plugin workers, run on demand

packages/metro-file-map/API.md

@@ -119,6 +119,26 @@ export class DuplicateHasteCandidatesError extends Error {
 
 export type FileData = Map<CanonicalPath, FileMetadata>;
 
+export class FileDataPlugin<PerFileData extends void | V8Serializable = void | V8Serializable> implements FileMapPlugin<null, PerFileData> {
+  constructor($$PARAM_0$$: FileDataPluginOptions);
+  assertValid(): void;
+  getCacheKey(): string;
+  getFileSystem(): FileMapPluginInitOptions<null, PerFileData>['files'];
+  getSerializableSnapshot(): null;
+  getWorker(): FileMapPluginWorker;
+  initialize(initOptions: FileMapPluginInitOptions<null, PerFileData>): Promise<void>;
+  readonly name: string;
+  onChanged(_changes: ReadonlyFileSystemChanges<null | undefined | PerFileData>): void;
+  processFile(mixedPath: string): ReturnType<FileMapPluginInitOptions<null, PerFileData>['processFile']>;
+}
+
+export type FileDataPluginOptions = Readonly<
+  Omit<FileMapPluginWorker, 'name' | 'cacheKey'> & {
+    name: string;
+    cacheKey: string;
+  }
+>;
+
 class FileMap extends EventEmitter {
   constructor(options: InputOptions);
   build(): Promise<BuildResult>;

packages/metro-file-map/src/__tests__/index-test.js

@@ -107,7 +107,6 @@ jest.mock('../crawlers/watchman', () => ({
               0, // visited
               hash,
               typeof contentOrLink !== 'string' ? 1 : 0,
-              null, // Haste name
             ]);
           }
         } else {
@@ -1707,6 +1706,64 @@ describe('FileMap', () => {
       );
     }
 
+    test('a lazy plugin processes files on demand, and its data is reset when the file changes', async () => {
+      const FileDataPlugin = require('../plugins/FileDataPlugin').default;
+      const lazyPlugin = new FileDataPlugin<?{length: number}>({
+        name: 'lazy-test-plugin',
+        cacheKey: 'lazy-test-plugin-1',
+        worker: {
+          modulePath: require.resolve('./lazy_plugin_worker.js'),
+          setupArgs: {},
+        },
+        // Not consulted for a lazy plugin
+        filter: () => true,
+        lazy: true,
+      });
+      const fileMap = new FileMap({
+        ...defaultConfig,
+        watch: true,
+        plugins: [lazyPlugin],
+      });
+      await fileMap.build();
+      try {
+        const bananaPath = path.join('/', 'project', 'fruits', 'Banana.js');
+        const getPluginData = () => {
+          const result = lazyPlugin.getFileSystem().lookup(bananaPath);
+          if (!result.exists || result.type !== 'f') {
+            throw new Error('Expected a file');
+          }
+          return result.pluginData;
+        };
+        const onMetadata = jest.fn();
+        fileMap.on('metadata', onMetadata);
+
+        // The crawl does not run a lazy plugin's worker
+        expect(getPluginData()).toBeUndefined();
+
+        const expected = {length: String(mockFs[bananaPath]).length};
+        expect(lazyPlugin.processFile(bananaPath)).toEqual(expected);
+        // The data is stored, and the cache told there is something to save
+        expect(getPluginData()).toEqual(expected);
+        expect(onMetadata).toHaveBeenCalledTimes(1);
+
+        mockFs[bananaPath] = '// A changed banana';
+        mockEmitters[path.join('/', 'project', 'fruits')].emitFileEvent({
+          event: 'touch',
+          relativePath: 'Banana.js',
+          metadata: MOCK_CHANGE_FILE,
+        });
+        await waitForItToChange(fileMap);
+
+        // Nor does a change, which leaves the file unprocessed again
+        expect(getPluginData()).toBeUndefined();
+        expect(lazyPlugin.processFile(bananaPath)).toEqual({
+          length: '// A changed banana'.length,
+        });
+      } finally {
+        await fileMap.end();
+      }
+    });
+
     function mockDeleteFile(root: string, relativePath: string) {
       const e = mockEmitters[root];
       e.emitFileEvent({event: 'delete', relativePath});

packages/metro-file-map/src/__tests__/lazy_plugin_worker.js

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ * @oncall react_native
+ */
+
+/* eslint-disable import/no-commonjs */
+
+'use strict';
+
+/*::
+import type {MetadataWorker, WorkerMessage, V8Serializable} from '../flow-types';
+*/
+
+// A plugin worker for tests, whose data for a file is the length of its content.
+module.exports = class LazyPluginWorker /*:: implements MetadataWorker */ {
+  processFile(
+    data /*: WorkerMessage */,
+    utils /*: Readonly<{getContent: () => Buffer }> */,
+  ) /*: V8Serializable */ {
+    return {length: utils.getContent().toString().length};
+  }
+};

packages/metro-file-map/src/crawlers/__tests__/integration-test.js

@@ -76,29 +76,23 @@ const CASES = [
   [
     true,
     new Map([
-      ['foo.js', [expect.any(Number), 245, 0, null, 0, null]],
-      [
-        join('directory', 'bar.js'),
-        [expect.any(Number), 245, 0, null, 0, null],
-      ],
+      ['foo.js', [expect.any(Number), 245, 0, null, 0]],
+      [join('directory', 'bar.js'), [expect.any(Number), 245, 0, null, 0]],
       [
         'link-to-directory',
-        [expect.any(Number), 9, 0, null, expect.oneOf(1, 'directory'), null],
+        [expect.any(Number), 9, 0, null, expect.oneOf(1, 'directory')],
       ],
       [
         'link-to-foo.js',
-        [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js'), null],
+        [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js')],
       ],
     ]),
   ],
   [
     false,
     new Map([
-      [
-        join('directory', 'bar.js'),
-        [expect.any(Number), 245, 0, null, 0, null],
-      ],
-      ['foo.js', [expect.any(Number), 245, 0, null, 0, null]],
+      [join('directory', 'bar.js'), [expect.any(Number), 245, 0, null, 0]],
+      ['foo.js', [expect.any(Number), 245, 0, null, 0]],
     ]),
   ],
 ];
@@ -118,7 +112,7 @@ describe.each(Object.keys(CRAWLERS))(
           previousState: {
             fileSystem: new TreeFS({
               rootDir: FIXTURES_DIR,
-              files: new Map([['removed.js', [123, 234, 0, null, 0, null]]]),
+              files: new Map([['removed.js', [123, 234, 0, null, 0]]]),
               processFile: () => {
                 throw new Error('Not implemented');
               },

packages/metro-file-map/src/crawlers/__tests__/node-test.js

@@ -130,7 +130,7 @@ describe('node crawler', () => {
     // Tomato is not included because its mtime is unchanged
     expect(changedFiles).toEqual(
       createMap({
-        'fruits/directory/strawberry.js': [33, 42, 0, null, 0, null],
+        'fruits/directory/strawberry.js': [33, 42, 0, null, 0],
       }),
     );
 
@@ -144,8 +144,8 @@ describe('node crawler', () => {
     // be found when crawling this directory.
     const files = createMap({
       'fruits/previouslyExisted.js': [30, 40, 1, null, 0, null],
-      'fruits/directory/strawberry.js': [33, 42, 0, null, 0, null],
-      'fruits/tomato.js': [32, 42, 0, null, 0, null],
+      'fruits/directory/strawberry.js': [33, 42, 0, null, 0],
+      'fruits/tomato.js': [32, 42, 0, null, 0],
     });
 
     const {changedFiles, removedFiles} = await nodeCrawl({
@@ -216,8 +216,8 @@ describe('node crawler', () => {
 
     expect(changedFiles).toEqual(
       createMap({
-        'fruits/directory/strawberry.js': [33, 42, 0, null, 0, null],
-        'fruits/tomato.js': [32, 42, 0, null, 0, null],
+        'fruits/directory/strawberry.js': [33, 42, 0, null, 0],
+        'fruits/tomato.js': [32, 42, 0, null, 0],
       }),
     );
     expect(removedFiles).toEqual(new Set());

packages/metro-file-map/src/crawlers/__tests__/watchman-test.js

@@ -118,9 +118,9 @@ describe('watchman watch', () => {
     };
 
     mockFiles = createMap({
-      [MELON_RELATIVE]: [33, 43, 0, null, 0, null],
-      [STRAWBERRY_RELATIVE]: [30, 40, 0, null, 0, null],
-      [TOMATO_RELATIVE]: [31, 41, 0, null, 0, null],
+      [MELON_RELATIVE]: [33, 43, 0, null, 0],
+      [STRAWBERRY_RELATIVE]: [30, 40, 0, null, 0],
+      [TOMATO_RELATIVE]: [31, 41, 0, null, 0],
     });
   });
 
@@ -223,7 +223,7 @@ describe('watchman watch', () => {
 
     expect(changedFiles).toEqual(
       createMap({
-        [KIWI_RELATIVE]: [42, 40, 0, null, 0, null],
+        [KIWI_RELATIVE]: [42, 40, 0, null, 0],
       }),
     );
 
@@ -296,7 +296,7 @@ describe('watchman watch', () => {
     // banana is not included because it is unchanged
     expect(changedFiles).toEqual(
       createMap({
-        [KIWI_RELATIVE]: [42, 52, 0, null, 0, null],
+        [KIWI_RELATIVE]: [42, 52, 0, null, 0],
         [TOMATO_RELATIVE]: [76, 41, 1, mockTomatoSha1, 0, 'Tomato'],
       }),
     );
@@ -373,7 +373,7 @@ describe('watchman watch', () => {
     // Melon is not included because it is unchanged.
     expect(changedFiles).toEqual(
       createMap({
-        [KIWI_RELATIVE]: [42, 52, 0, null, 0, null],
+        [KIWI_RELATIVE]: [42, 52, 0, null, 0],
       }),
     );
 
@@ -542,7 +542,7 @@ describe('watchman watch', () => {
 
     expect(changedFiles).toEqual(
       createMap({
-        [KIWI_RELATIVE]: [42, 40, 0, null, 0, null],
+        [KIWI_RELATIVE]: [42, 40, 0, null, 0],
       }),
     );
 

packages/metro-file-map/src/crawlers/node/index.js

@@ -75,7 +75,6 @@ function find(
                   0,
                   null,
                   stat.isSymbolicLink() ? 1 : 0,
-                  null,
                 ]);
               }
             }

packages/metro-file-map/src/crawlers/watchman/index.js

@@ -334,7 +334,6 @@ export default async function watchmanCrawl({
           0,
           sha1hex ?? null,
           symlinkInfo,
-          null,
         ];
 
         // If watchman is fresh, the removed files map starts with all files

packages/metro-file-map/src/flow-types.js

@@ -222,6 +222,18 @@ export type FileMapPluginInitOptions<
       | {exists: true, type: 'd'},
   }>,
   pluginState: ?SerializableState,
+  /**
+   * Synchronously run this plugin's worker, in-band, on the regular file at
+   * `mixedPath`, store the result as its plugin data and return it. Throws if
+   * the path is not a regular file, or the plugin has no worker.
+   *
+   * Plugin data is `undefined` for a file until a worker has run on it, which
+   * is when a lazy plugin should call this, and is reset to `undefined` when
+   * the file changes. `null` is a result like any other, so what a worker
+   * returns is stored with `undefined` replaced by `null`, and a file is
+   * processed at most once while it is unchanged.
+   */
+  processFile: (mixedPath: string) => PerFileData,
 }>;
 
 export type FileMapPluginWorker = Readonly<{
@@ -230,6 +242,13 @@ export type FileMapPluginWorker = Readonly<{
     setupArgs: JsonData,
   }>,
   filter: ({normalPath: string, isNodeModules: boolean}) => boolean,
+  /**
+   * If true, the worker is not run on files as they are crawled or changed,
+   * and `filter` is not consulted. It runs only on the files the plugin asks
+   * for, through the `processFile` it is given on initialization. Use this
+   * where a small, unpredictable subset of matching files is ever needed.
+   */
+  lazy?: boolean,
 }>;
 
 type V8SerializablePrimitive = string | number | boolean | null;
@@ -293,7 +312,7 @@ export type FileMetadata = [
   /* visited */ 0 | 1,
   /* sha1 */ ?string,
   /* symlink */ 0 | 1 | string, // string specifies target, if known
-  /* plugindata */
+  /* plugindata - one slot per plugin with a worker, `undefined` until run */
   ...
 ];
 

packages/metro-file-map/src/index.js

@@ -142,6 +142,8 @@ type InternalEnqueuedEvent = Readonly<
 export {DiskCacheManager} from './cache/DiskCacheManager';
 export {NoopCacheManager} from './cache/NoopCacheManager';
 export {default as DependencyPlugin} from './plugins/DependencyPlugin';
+export {default as FileDataPlugin} from './plugins/FileDataPlugin';
+export type {FileDataPluginOptions} from './plugins/FileDataPlugin';
 export type {DependencyPluginOptions} from './plugins/DependencyPlugin';
 export {DuplicateHasteCandidatesError} from './plugins/haste/DuplicateHasteCandidatesError';
 export {HasteConflictsError} from './plugins/haste/HasteConflictsError';
@@ -472,6 +474,32 @@ export default class FileMap extends EventEmitter {
                     ),
                 },
                 pluginState: initialData?.plugins.get(plugin.name),
+                processFile: mixedPath => {
+                  invariant(
+                    dataIdx != null,
+                    'metro-file-map: Plugin "%s" has no worker to process files with',
+                    plugin.name,
+                  );
+                  const result = fileSystem.lookup(mixedPath);
+                  if (!result.exists || result.type !== 'f') {
+                    throw new Error(
+                      `metro-file-map: Cannot process ${mixedPath}, which is not a regular file`,
+                    );
+                  }
+                  const pluginData = this.#fileProcessor.processFileForPlugin(
+                    result.realPath,
+                    result.metadata,
+                    dataIdx - H.PLUGINDATA,
+                  );
+                  debug(
+                    'Lazily processed file for %s: %s',
+                    plugin.name,
+                    mixedPath,
+                  );
+                  // Inform caches that there is new data to save.
+                  this.emit('metadata');
+                  return pluginData;
+                },
               }),
             ),
           ),
@@ -928,7 +956,6 @@ export default class FileMap extends EventEmitter {
               0,
               null,
               change.metadata.type === 'l' ? 1 : 0,
-              null,
             ];
 
             try {