fix: "Can't reconcile two non-macho files" due to new Pre-Gyp-Copy functionality in electron/rebuild integration (#7519)

Adding native module to two-package test fixture.
Ignoring electron/rebuild metadata and disabling its pregyp cache
Exclude .husky and .github folders by default.
Verify smart unpack for universal builds with native module (node-mac-permissions in this case)

Author: mmaietta

.changeset/flat-jars-glow.md

@@ -0,0 +1,6 @@
+---
+"app-builder-lib": patch
+"electron-builder": patch
+---
+
+fix: "Can't reconcile two non-macho files" due to `disablePreGypCopy` functionality in new electron/rebuild integration

packages/app-builder-lib/src/fileMatcher.ts

@@ -17,9 +17,12 @@ export const excludedNames =
   "__pycache__,.DS_Store,thumbs.db,.gitignore,.gitkeep,.gitattributes,.npmignore," +
   ".idea,.vs,.flowconfig,.jshintrc,.eslintrc,.circleci," +
   ".yarn-integrity,.yarn-metadata.json,yarn-error.log,yarn.lock,package-lock.json,npm-debug.log," +
-  "appveyor.yml,.travis.yml,circle.yml,.nyc_output"
+  "appveyor.yml,.travis.yml,circle.yml,.nyc_output,.husky,.github"
 
-export const excludedExts = "iml,hprof,orig,pyc,pyo,rbc,swp,csproj,sln,suo,xproj,cc,d.ts"
+export const excludedExts =
+  "iml,hprof,orig,pyc,pyo,rbc,swp,csproj,sln,suo,xproj,cc,d.ts," +
+  // https://github.com/electron-userland/electron-builder/issues/7512
+  "mk,a,o,forge-meta"
 
 function ensureNoEndSlash(file: string): string {
   if (path.sep !== "/") {

packages/app-builder-lib/src/macPackager.ts

@@ -106,11 +106,13 @@ export default class MacPackager extends PlatformPackager<MacConfiguration> {
         return super.doPack(outDir, appOutDir, platformName, arch, platformSpecificBuildOptions, targets)
       }
       case Arch.universal: {
+        const outDirName = (arch: Arch) => `${appOutDir}-${Arch[arch]}-temp`
+
         const x64Arch = Arch.x64
-        const x64AppOutDir = appOutDir + "--" + Arch[x64Arch]
+        const x64AppOutDir = outDirName(x64Arch)
         await super.doPack(outDir, x64AppOutDir, platformName, x64Arch, platformSpecificBuildOptions, targets, false, true)
         const arm64Arch = Arch.arm64
-        const arm64AppOutPath = appOutDir + "--" + Arch[arm64Arch]
+        const arm64AppOutPath = outDirName(arm64Arch)
         await super.doPack(outDir, arm64AppOutPath, platformName, arm64Arch, platformSpecificBuildOptions, targets, false, true)
         const framework = this.info.framework
         log.info(

packages/app-builder-lib/src/packager.ts

@@ -495,7 +495,7 @@ export class Packager {
     const frameworkInfo = { version: this.framework.version, useCustomDist: true }
     const config = this.config
     if (config.nodeGypRebuild === true) {
-      await nodeGypRebuild(frameworkInfo, Arch[arch])
+      await nodeGypRebuild(frameworkInfo, arch)
     }
 
     if (config.npmRebuild === false) {
@@ -526,7 +526,6 @@ export class Packager {
         frameworkInfo,
         platform: platform.nodeName,
         arch: Arch[arch],
-        productionDeps: this.getNodeDependencyInfo(null),
       })
     }
   }

packages/app-builder-lib/src/util/yarn.ts

@@ -1,10 +1,8 @@
-import { asArray, log, spawn } from "builder-util"
+import { Arch, archFromString, asArray, log, spawn } from "builder-util"
 import { pathExists } from "fs-extra"
-import { Lazy } from "lazy-val"
 import { homedir } from "os"
 import * as path from "path"
 import { Configuration } from "../configuration"
-import { NodeModuleDirInfo } from "./packageDependencies"
 import * as electronRebuild from "@electron/rebuild"
 import * as searchModule from "@electron/rebuild/lib/src/search-module"
 
@@ -25,7 +23,8 @@ export async function installOrRebuild(config: Configuration, appDir: string, op
     }
     await installDependencies(appDir, effectiveOptions)
   } else {
-    await rebuild(appDir, config.buildDependenciesFromSource === true, options.frameworkInfo, options.arch)
+    const arch = archFromString(options.arch || process.arch)
+    await rebuild(appDir, config.buildDependenciesFromSource === true, options.frameworkInfo, arch)
   }
 }
 
@@ -118,7 +117,7 @@ function installDependencies(appDir: string, options: RebuildOptions): Promise<a
   })
 }
 
-export async function nodeGypRebuild(frameworkInfo: DesktopFrameworkInfo, arch: string) {
+export async function nodeGypRebuild(frameworkInfo: DesktopFrameworkInfo, arch: Arch) {
   return rebuild(process.cwd(), false, frameworkInfo, arch)
 }
 
@@ -137,7 +136,6 @@ function isRunningYarn(execPath: string | null | undefined) {
 
 export interface RebuildOptions {
   frameworkInfo: DesktopFrameworkInfo
-  productionDeps?: Lazy<Array<NodeModuleDirInfo>>
 
   platform?: NodeJS.Platform
   arch?: string
@@ -148,15 +146,15 @@ export interface RebuildOptions {
 }
 
 /** @internal */
-export async function rebuild(appDir: string, buildFromSource: boolean, frameworkInfo: DesktopFrameworkInfo, arch = process.arch) {
-  log.info({ arch, version: frameworkInfo.version, appDir }, "executing @electron/rebuild")
+export async function rebuild(appDir: string, buildFromSource: boolean, frameworkInfo: DesktopFrameworkInfo, arch: Arch) {
+  log.info({ arch: Arch[arch], version: frameworkInfo.version, appDir }, "executing @electron/rebuild")
   const rootPath = await searchModule.getProjectRootPath(appDir)
   const options: electronRebuild.RebuildOptions = {
     buildPath: appDir,
     electronVersion: frameworkInfo.version,
-    arch,
-    debug: log.isDebugEnabled,
+    arch: Arch[arch],
     projectRootPath: rootPath,
+    disablePreGypCopy: true,
   }
   if (buildFromSource) {
     options.prebuildTagPrefix = "totally-not-a-real-prefix-to-force-rebuild"

packages/electron-builder/src/cli/install-app-deps.ts

@@ -1,12 +1,11 @@
 #! /usr/bin/env node
 
-import { PACKAGE_VERSION } from "app-builder-lib/out/version"
-import { log, use, getArchCliNames } from "builder-util"
-import { printErrorAndExit } from "builder-util/out/promise"
-import { computeDefaultAppDirectory, getConfig } from "app-builder-lib/out/util/config"
 import { getElectronVersion } from "app-builder-lib/out/electron/electronVersion"
-import { createLazyProductionDeps } from "app-builder-lib/out/util/packageDependencies"
+import { computeDefaultAppDirectory, getConfig } from "app-builder-lib/out/util/config"
 import { installOrRebuild } from "app-builder-lib/out/util/yarn"
+import { PACKAGE_VERSION } from "app-builder-lib/out/version"
+import { getArchCliNames, log, use } from "builder-util"
+import { printErrorAndExit } from "builder-util/out/promise"
 import { readJson } from "fs-extra"
 import { Lazy } from "lazy-val"
 import * as path from "path"
@@ -63,7 +62,6 @@ export async function installAppDeps(args: any) {
       frameworkInfo: { version, useCustomDist: true },
       platform: args.platform,
       arch: args.arch,
-      productionDeps: createLazyProductionDeps(appDir, null),
     },
     appDir !== projectDir
   )

test/fixtures/test-app-two-native-modules/app/package.json

@@ -3,6 +3,9 @@
   "main": "index.js",
   "version": "1.1.1",
   "dependencies": {
-    "install": "0.13.0"
+    "debug": "4.1.1"
+  },
+  "optionalDependencies": {
+    "node-mac-permissions": "^2.3.0"
   }
 }

test/snapshots/BuildTest.js.snap

@@ -957,13 +957,6 @@ Object {
         },
         "minimist": Object {
           "files": Object {
-            ".github": Object {
-              "files": Object {
-                "FUNDING.yml": Object {
-                  "size": "<size>",
-                },
-              },
-            },
             ".nycrc": Object {
               "size": "<size>",
             },

test/snapshots/ExtraBuildTest.js.snap …napshots/ExtraBuildResourcesTest.js.snaptest/snapshots/ExtraBuildTest.js.snap renamed to test/snapshots/ExtraBuildResourcesTest.js.snap test/snapshots/ExtraBuildTest.js.snap renamed to test/snapshots/ExtraBuildResourcesTest.js.snap

@@ -185,45 +185,67 @@ Object {
     },
     "node_modules": Object {
       "files": Object {
-        "install": Object {
+        "debug": Object {
           "files": Object {
             "LICENSE": Object {
-              "offset": "3479",
-              "size": 1101,
+              "offset": "3546",
+              "size": 1107,
             },
-            "install.js": Object {
-              "offset": "4580",
-              "size": 19880,
-            },
-            "install.min.js": Object {
-              "offset": "24460",
-              "size": 3663,
+            "dist": Object {
+              "files": Object {
+                "debug.js": Object {
+                  "offset": "22150",
+                  "size": 27572,
+                },
+              },
             },
             "package.json": Object {
-              "offset": "28123",
-              "size": 529,
+              "offset": "4653",
+              "size": 947,
             },
-            "scripts": Object {
+            "src": Object {
               "files": Object {
-                "docs.sh": Object {
-                  "executable": true,
-                  "offset": "28652",
-                  "size": 99,
+                "browser.js": Object {
+                  "offset": "5600",
+                  "size": 5831,
                 },
-                "prepublish.sh": Object {
-                  "executable": true,
-                  "offset": "28751",
-                  "size": 83,
+                "common.js": Object {
+                  "offset": "11431",
+                  "size": 5930,
+                },
+                "index.js": Object {
+                  "offset": "17361",
+                  "size": 314,
+                },
+                "node.js": Object {
+                  "offset": "17675",
+                  "size": 4475,
                 },
               },
             },
           },
         },
+        "ms": Object {
+          "files": Object {
+            "index.js": Object {
+              "offset": "49722",
+              "size": 3024,
+            },
+            "license.md": Object {
+              "offset": "52746",
+              "size": 1079,
+            },
+            "package.json": Object {
+              "offset": "53825",
+              "size": 497,
+            },
+          },
+        },
       },
     },
     "package.json": Object {
       "offset": "3342",
-      "size": 137,
+      "size": 204,
     },
   },
 }