feat(compartment-mapper): allow force-loading of compartments

This adds option `forceLoad` to `captureFromMap()`. This is an optional `string[]` of keys of the `CompartmentMapDescriptor` also provided to `captureFromMap()`.

After loading the entry Compartment and attenuators Compartment, the Compartments having names (practically speaking, these will be _locations_) mentioned in `forceLoad` will then be loaded _if they were not already_.

This option can be used to support dynamic requires and imports which would otherwise be omitted from the captured `CompartmentMapDescriptor` via digestion.

* * *

- Refactored `capture-lite.js`; stuffed all of the Compartment-loading business into a function.
- Added test and fixture that shows how a `Compartment` which would otherwise be omitted from the captured Compartment Map is included when used with `forceLoad`.

Author: boneskull

packages/compartment-mapper/src/capture-lite.js

@@ -36,25 +36,31 @@
  *   CaptureLiteOptions,
  *   CaptureResult,
  *   CompartmentMapDescriptor,
+ *   ForceLoadOption,
+ *   LogFn,
+ *   LogOptions,
+ *   PolicyOption,
  *   ReadFn,
  *   ReadPowers,
  *   Sources,
  * } from './types.js'
  */
 
+import { digestCompartmentMap } from './digest.js';
 import {
   exitModuleImportHookMaker,
   makeImportHookMaker,
 } from './import-hook.js';
 import { link } from './link.js';
 import { resolve } from './node-module-specifier.js';
+import { ATTENUATORS_COMPARTMENT } from './policy-format.js';
 import { detectAttenuators } from './policy.js';
 import { unpackReadPowers } from './powers.js';
-import { digestCompartmentMap } from './digest.js';
 
-const { freeze, assign, create } = Object;
+const { freeze, assign, create, keys } = Object;
+const { stringify: q } = JSON;
 
-const defaultCompartment = Compartment;
+const DefaultCompartment = Compartment;
 
 /**
  * @param {CompartmentMapDescriptor} compartmentMap
@@ -79,12 +85,160 @@ const captureCompartmentMap = (compartmentMap, sources) => {
 };
 
 /**
- * @param {ReadFn | ReadPowers} powers
+ * @type {LogFn}
+ */
+const noop = () => {};
+
+/**
+ * Factory for a function that loads compartments.
+ *
+ * @param {CompartmentMapDescriptor} compartmentMap Compartment map
+ * @param {Sources} sources Sources
+ * @param {LogOptions & PolicyOption & ForceLoadOption} [options]
+ * @returns {(linkedCompartments: Record<string, Compartment>, entryCompartment: Compartment, attenuatorsCompartment: Compartment) => Promise<void>}
+ */
+const makeLoadCompartments = (
+  compartmentMap,
+  sources,
+  { log = noop, policy, forceLoad = [] } = {},
+) => {
+  const {
+    entry: { module: entryModuleSpecifier },
+  } = compartmentMap;
+
+  /**
+   * Given {@link CompartmentDescriptor CompartmentDescriptors}, loads any which a) are present in the {@link forceLoad forceLoad array}, and b) have not yet been loaded.
+   *
+   * Will not load the "attenuators" `Compartment`, nor will it load any
+   * `Compartment` having a non-empty value in `sources` (since it is presumed it
+   * has already been loaded).
+   *
+   * @param {Record<string, Compartment>} compartments
+   * @returns {Promise<void>} Resolves when all appropriate compartments are loaded.
+   */
+  const forceLoadCompartments = async compartments => {
+    const compartmentsToLoad = forceLoad.reduce((acc, compartmentName) => {
+      // skip; should already be loaded
+      if (
+        compartmentName === ATTENUATORS_COMPARTMENT ||
+        compartmentName === compartmentMap.entry.compartment
+      ) {
+        return acc;
+      }
+
+      const compartmentDescriptor =
+        compartmentMap.compartments[compartmentName];
+
+      if (!compartmentDescriptor) {
+        throw new ReferenceError(
+          `Failed attempting to force-load unknown compartment ${q(compartmentName)}`,
+        );
+      }
+
+      const compartmentSources = sources[compartmentName];
+
+      if (keys(compartmentSources).length) {
+        log(
+          `Refusing to force-load Compartment ${q(compartmentName)}; already loaded`,
+        );
+        return acc;
+      }
+
+      const compartment = compartments[compartmentName];
+      if (!compartment) {
+        throw new ReferenceError(
+          `No compartment found for ${q(compartmentName)}`,
+        );
+      }
+      const compartmentOwnModuleDescriptor =
+        compartmentDescriptor.modules[compartmentDescriptor.name];
+
+      if (!compartmentOwnModuleDescriptor?.module) {
+        throw new Error(
+          `Cannot determine entry point of ${q(compartmentName)}`,
+        );
+      }
+      acc.push([
+        compartmentName,
+        compartment,
+        compartmentOwnModuleDescriptor.module,
+      ]);
+
+      // could delete the `load` flag here, but it gets dropped during
+      // capture anyway
+      return acc;
+    }, /** @type {[compartmentName: string, compartment: Compartment, moduleSpecifier: string][]} */ ([]));
+
+    const { length: compartmentsToLoadCount } = compartmentsToLoad;
+    /**
+     * This index increments in the order in which compartments finish
+     * loading—_not_ the order in which they began loading.
+     */
+    let loadedCompartmentIndex = 0;
+    await Promise.all(
+      compartmentsToLoad.map(
+        async ([compartmentName, compartment, moduleSpecifier]) => {
+          await compartment.load(moduleSpecifier);
+          log(
+            `Force-loaded Compartment: ${q(compartmentName)} (${(loadedCompartmentIndex += 1)}/${compartmentsToLoadCount})`,
+          );
+        },
+      ),
+    );
+  };
+
+  /**
+   * Loads, in order:
+   *
+   * 1. The entry compartment
+   * 2. The attenuators compartment (_if and only if_ `policy` was provided)
+   * 3. All compartments in the `compartmentMap` that have the `load` bit set.
+   *
+   * @param {Record<string, Compartment>} linkedCompartments
+   * @param {Compartment} entryCompartment
+   * @param {Compartment} attenuatorsCompartment
+   * @returns {Promise<void>} Resolves when all compartments are loaded.
+   */
+  const loadCompartments = async (
+    linkedCompartments,
+    entryCompartment,
+    attenuatorsCompartment,
+  ) => {
+    await entryCompartment.load(entryModuleSpecifier);
+
+    if (policy) {
+      // retain all attenuators.
+      await Promise.all(
+        detectAttenuators(policy).map(attenuatorSpecifier =>
+          attenuatorsCompartment.load(attenuatorSpecifier),
+        ),
+      );
+    }
+
+    await forceLoadCompartments(linkedCompartments);
+  };
+
+  return loadCompartments;
+};
+
+/**
+ * "Captures" the compartment map descriptors and sources from a partially
+ * completed compartment map—_without_ creating an archive.
+ *
+ * The resulting compartment map represents a well-formed dependency graph,
+ * laden with useful metadata. This, for example, could be used for automatic
+ * policy generation.
+ *
+ * @param {ReadFn | ReadPowers} readPowers Powers
  * @param {CompartmentMapDescriptor} compartmentMap
  * @param {CaptureLiteOptions} [options]
  * @returns {Promise<CaptureResult>}
  */
-export const captureFromMap = async (powers, compartmentMap, options = {}) => {
+export const captureFromMap = async (
+  readPowers,
+  compartmentMap,
+  options = {},
+) => {
   const {
     moduleTransforms,
     syncModuleTransforms,
@@ -94,14 +248,15 @@ export const captureFromMap = async (powers, compartmentMap, options = {}) => {
     policy = undefined,
     sourceMapHook = undefined,
     parserForLanguage: parserForLanguageOption = {},
-    Compartment = defaultCompartment,
+    Compartment: CompartmentOption = DefaultCompartment,
+    log = noop,
+    forceLoad = [],
   } = options;
-
   const parserForLanguage = freeze(
     assign(create(null), parserForLanguageOption),
   );
 
-  const { read, computeSha512 } = unpackReadPowers(powers);
+  const { read, computeSha512 } = unpackReadPowers(readPowers);
 
   const {
     compartments,
@@ -111,6 +266,12 @@ export const captureFromMap = async (powers, compartmentMap, options = {}) => {
   /** @type {Sources} */
   const sources = Object.create(null);
 
+  const loadCompartments = makeLoadCompartments(compartmentMap, sources, {
+    log,
+    policy,
+    forceLoad,
+  });
+
   const consolidatedExitModuleImportHook = exitModuleImportHookMaker({
     modules: exitModules,
     exitModuleImportHook,
@@ -128,25 +289,27 @@ export const captureFromMap = async (powers, compartmentMap, options = {}) => {
     importHook: consolidatedExitModuleImportHook,
     sourceMapHook,
   });
+
   // Induce importHook to record all the necessary modules to import the given module specifier.
-  const { compartment, attenuatorsCompartment } = link(compartmentMap, {
+  const {
+    compartment: entryCompartment,
+    compartments: linkedCompartments,
+    attenuatorsCompartment,
+  } = link(compartmentMap, {
     resolve,
     makeImportHook,
     moduleTransforms,
     syncModuleTransforms,
     parserForLanguage,
     archiveOnly: true,
-    Compartment,
+    Compartment: CompartmentOption,
   });
-  await compartment.load(entryModuleSpecifier);
-  if (policy) {
-    // retain all attenuators.
-    await Promise.all(
-      detectAttenuators(policy).map(attenuatorSpecifier =>
-        attenuatorsCompartment.load(attenuatorSpecifier),
-      ),
-    );
-  }
+
+  await loadCompartments(
+    linkedCompartments,
+    entryCompartment,
+    attenuatorsCompartment,
+  );
 
   return captureCompartmentMap(compartmentMap, sources);
 };

packages/compartment-mapper/src/types/external.ts

@@ -135,10 +135,18 @@ export type CompartmentMapForNodeModulesOptions = Omit<
   'conditions' | 'tags'
 >;
 
+/**
+ * Options for `captureFromMap()`
+ */
 export type CaptureLiteOptions = ImportingOptions &
   LinkingOptions &
   PolicyOption &
-  LogOptions;
+  LogOptions &
+  ForceLoadOption;
+
+export interface ForceLoadOption {
+  forceLoad?: Array<string>;
+}
 
 export type ArchiveLiteOptions = SyncOrAsyncArchiveOptions &
   ModuleTransformsOption &

packages/compartment-mapper/test/capture-lite.test.js

@@ -1,4 +1,5 @@
 import 'ses';
+
 import fs from 'node:fs';
 import url from 'node:url';
 import path from 'node:path';
@@ -10,7 +11,7 @@ import { defaultParserForLanguage } from '../src/import-parsers.js';
 
 const { keys } = Object;
 
-test('captureFromMap() should resolve with a CaptureResult', async t => {
+test('captureFromMap() - should resolve with a CaptureResult', async t => {
   t.plan(5);
 
   const readPowers = makeReadPowers({ fs, url });
@@ -60,7 +61,73 @@ test('captureFromMap() should resolve with a CaptureResult', async t => {
   );
 });
 
-test('captureFromMap() should round-trip sources based on parsers', async t => {
+test('captureFromMap() - should discard unretained CompartmentDescriptors', async t => {
+  const readPowers = makeReadPowers({ fs, url });
+  const moduleLocation = `${new URL(
+    'fixtures-digest/node_modules/app/index.js',
+    import.meta.url,
+  )}`;
+
+  const nodeCompartmentMap = await mapNodeModules(readPowers, moduleLocation);
+
+  const nodeComartmentMapSize = keys(nodeCompartmentMap.compartments).length;
+
+  const { captureCompartmentMap } = await captureFromMap(
+    readPowers,
+    nodeCompartmentMap,
+    {
+      parserForLanguage: defaultParserForLanguage,
+    },
+  );
+
+  const captureCompartmentMapSize = keys(
+    captureCompartmentMap.compartments,
+  ).length;
+
+  t.true(
+    captureCompartmentMapSize < nodeComartmentMapSize,
+    'captureCompartmentMap should contain fewer CompartmentDescriptors than nodeCompartmentMap',
+  );
+
+  t.false(
+    'fjord-v1.0.0' in captureCompartmentMap.compartments,
+    '"fjord-v1.0.0" should not be retained in captureCompartmentMap',
+  );
+});
+
+test('captureFromMap() - should force-load', async t => {
+  const readPowers = makeReadPowers({ fs, url });
+  const moduleLocation = `${new URL(
+    'fixtures-digest/node_modules/app/index.js',
+    import.meta.url,
+  )}`;
+
+  const nodeCompartmentMap = await mapNodeModules(readPowers, moduleLocation);
+
+  const fjordCompartment = Object.values(nodeCompartmentMap.compartments).find(
+    c => c.name === 'fjord',
+  );
+  if (!fjordCompartment) {
+    t.fail('Expected "fjord" compartment to be present in nodeCompartmentMap');
+    return;
+  }
+
+  const { captureCompartmentMap } = await captureFromMap(
+    readPowers,
+    nodeCompartmentMap,
+    {
+      forceLoad: [fjordCompartment.location],
+      parserForLanguage: defaultParserForLanguage,
+    },
+  );
+
+  t.true(
+    'fjord-v1.0.0' in captureCompartmentMap.compartments,
+    '"fjord-v1.0.0" should be retained in captureCompartmentMap',
+  );
+});
+
+test('captureFromMap() - should round-trip sources based on parsers', async t => {
   const readPowers = makeReadPowers({ fs, url });
   const moduleLocation = `${new URL(
     'fixtures-0/node_modules/bundle/main.js',

packages/compartment-mapper/test/fixtures-digest/README.md

@@ -0,0 +1 @@
+This fixture illustrates a case where `mapNodeModules()` should return a `CompartmentMapDescriptor` having more compartments than the result of passing it thru `captureFromMap()`.