feat: yarn 3 support for native modules via new electron/rebuild compilation (#8112)

Author: mmaietta

.changeset/twelve-points-whisper.md

@@ -0,0 +1,5 @@
+---
+"app-builder-lib": minor
+---
+
+feat: implementing electron/rebuild with config option `useLegacyRebuilder` default: `true` to support Yarn 3

.github/actions/pretest/action.yml

@@ -20,6 +20,11 @@ runs:
       with:
         version: ${{ inputs.version }}
 
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
     - name: Setup node
       uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
       with:

docs/configuration/configuration.md

@@ -93,6 +93,9 @@ Env file `electron-builder.env` in the current dir ([example](https://github.com
 <p><code id="Configuration-npmRebuild">npmRebuild</code> = <code>true</code> Boolean - Whether to <a href="https://docs.npmjs.com/cli/rebuild">rebuild</a> native dependencies before starting to package the app.</p>
 </li>
 <li>
+<p><code id="Configuration-nativeRebuilder">nativeRebuilder</code> = <code>legacy</code> “legacy” | “sequential” | “parallel” | “undefined” - Use <code>legacy</code> app-builder binary for installing native dependencies, or @electron/rebuild in <code>sequential</code> or <code>parallel</code> compilation modes.</p>
+</li>
+<li>
 <p><code id="Configuration-buildNumber">buildNumber</code> String | “undefined” - The build number. Maps to the <code>--iteration</code> flag for builds using FPM on Linux. If not defined, then it will fallback to <code>BUILD_NUMBER</code> or <code>TRAVIS_BUILD_NUMBER</code> or <code>APPVEYOR_BUILD_NUMBER</code> or <code>CIRCLE_BUILD_NUM</code> or <code>BUILD_BUILDNUMBER</code> or <code>CI_PIPELINE_IID</code> env.</p>
 </li>
 </ul>

packages/app-builder-lib/package.json

@@ -49,6 +49,7 @@
     "@develar/schema-utils": "~2.6.5",
     "@electron/notarize": "2.2.1",
     "@electron/osx-sign": "1.0.5",
+    "@electron/rebuild": "3.6.0",
     "@electron/universal": "1.5.1",
     "@malept/flatpak-bundler": "^0.4.0",
     "@types/fs-extra": "9.0.13",

packages/app-builder-lib/scheme.json

@@ -7463,6 +7463,23 @@
         }
       ]
     },
+    "nativeRebuilder": {
+      "anyOf": [
+        {
+          "enum": [
+            "legacy",
+            "parallel",
+            "sequential"
+          ],
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": "legacy",
+      "description": "Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes."
+    },
     "nodeGypRebuild": {
       "default": false,
       "description": "Whether to execute `node-gyp rebuild` before starting to package the app.\n\nDon't [use](https://github.com/electron-userland/electron-builder/issues/683#issuecomment-241214075) [npm](http://electron.atom.io/docs/tutorial/using-native-node-modules/#using-npm) (neither `.npmrc`) for configuring electron headers. Use `electron-builder node-gyp-rebuild` instead.",

packages/app-builder-lib/src/configuration.ts

@@ -131,6 +131,11 @@ export interface Configuration extends PlatformSpecificBuildOptions {
    * @default true
    */
   readonly npmRebuild?: boolean
+  /**
+   * Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes.
+   * @default legacy
+   */
+  readonly nativeRebuilder?: "legacy" | "sequential" | "parallel" | null
 
   /**
    * The build number. Maps to the `--iteration` flag for builds using FPM on Linux.

packages/app-builder-lib/src/electron/electronVersion.ts

@@ -1,3 +1,5 @@
+import { getProjectRootPath } from "@electron/rebuild/lib/search-module"
+
 import { InvalidConfigurationError, log } from "builder-util"
 import { parseXml } from "builder-util-runtime"
 import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
@@ -56,8 +58,15 @@ export async function computeElectronVersion(projectDir: string): Promise<string
     return result
   }
 
-  const metadata = await orNullIfFileNotExist(readJson(path.join(projectDir, "package.json")))
-  const dependency = metadata ? findFromPackageMetadata(metadata) : null
+  const potentialRootDirs = [projectDir, await getProjectRootPath(projectDir)]
+  let dependency: NameAndVersion | null = null
+  for await (const dir of potentialRootDirs) {
+    const metadata = await orNullIfFileNotExist(readJson(path.join(dir, "package.json")))
+    dependency = metadata ? findFromPackageMetadata(metadata) : null
+    if (dependency) {
+      break
+    }
+  }
   if (dependency?.name === "electron-nightly") {
     log.info("You are using a nightly version of electron, be warned that those builds are highly unstable.")
     const feedXml = await httpExecutor.request({

packages/app-builder-lib/src/util/packageMetadata.ts

@@ -70,7 +70,7 @@ export function checkMetadata(metadata: Metadata, devMetadata: any | null, appPa
   const devDependencies = (metadata as any).devDependencies
   if (devDependencies != null && ("electron-rebuild" in devDependencies || "@electron/rebuild" in devDependencies)) {
     log.info(
-      '@electron/rebuild not required if you use electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
+      '@electron/rebuild already used by electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
     )
   }
 

packages/app-builder-lib/src/util/rebuild/rebuild.ts

@@ -0,0 +1,59 @@
+import * as cp from "child_process"
+import * as path from "path"
+import { RebuildOptions } from "@electron/rebuild"
+import { log } from "builder-util"
+
+export const rebuild = async (options: RebuildOptions): Promise<void> => {
+  const { arch } = options
+  log.info({ arch }, `installing native dependencies`)
+
+  const child = cp.fork(path.resolve(__dirname, "remote-rebuild.js"), [JSON.stringify(options)], {
+    stdio: ["pipe", "pipe", "pipe", "ipc"],
+  })
+
+  let pendingError: Error
+
+  child.stdout?.on("data", chunk => {
+    log.info(chunk.toString())
+  })
+  child.stderr?.on("data", chunk => {
+    log.error(chunk.toString())
+  })
+
+  child.on("message", (message: { msg: string; moduleName: string; err: { message: string; stack: string } }) => {
+    const { moduleName, msg } = message
+    switch (msg) {
+      case "module-found": {
+        log.info({ moduleName, arch }, "preparing")
+        break
+      }
+      case "module-done": {
+        log.info({ moduleName, arch }, "finished")
+        break
+      }
+      case "module-skip": {
+        log.debug?.({ moduleName, arch }, "skipped. set ENV=electron-rebuild to determine why")
+        break
+      }
+      case "rebuild-error": {
+        pendingError = new Error(message.err.message)
+        pendingError.stack = message.err.stack
+        break
+      }
+      case "rebuild-done": {
+        log.info("completed installing native dependencies")
+        break
+      }
+    }
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    child.on("exit", code => {
+      if (code === 0 && !pendingError) {
+        resolve()
+      } else {
+        reject(pendingError || new Error(`Rebuilder failed with exit code: ${code}`))
+      }
+    })
+  })
+}

packages/app-builder-lib/src/util/rebuild/remote-rebuild.ts

@@ -0,0 +1,30 @@
+import { rebuild, RebuildOptions } from "@electron/rebuild"
+
+if (!process.send) {
+  console.error("The remote rebuilder expects to be spawned with an IPC channel")
+  process.exit(1)
+}
+
+const options: RebuildOptions = JSON.parse(process.argv[2])
+
+const rebuilder = rebuild(options)
+
+rebuilder.lifecycle.on("module-found", (moduleName: string) => process.send?.({ msg: "module-found", moduleName }))
+rebuilder.lifecycle.on("module-done", (moduleName: string) => process.send?.({ msg: "module-done", moduleName }))
+rebuilder.lifecycle.on("module-skip", (moduleName: string) => process.send?.({ msg: "module-skip", moduleName }))
+
+rebuilder
+  .then(() => {
+    process.send?.({ msg: "rebuild-done" })
+    return process.exit(0)
+  })
+  .catch(err => {
+    process.send?.({
+      msg: "rebuild-error",
+      err: {
+        message: err.message,
+        stack: err.stack,
+      },
+    })
+    process.exit(0)
+  })