Skip to content

Commit 4edd695

Browse files
mmaiettammaietta
authored
chore: add logging summary to end of node collector to flag any special cases/alerts (#9587)
1 parent dde4309 commit 4edd695

File tree

10 files changed

+1634
-1571
lines changed

10 files changed

+1634
-1571
lines changed

‎.changeset/curvy-insects-notice.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"app-builder-lib": patch
3+
---
4+
5+
chore: add logging summary to end of node collector to flag any special cases/alerts

‎.changeset/nasty-trees-brush.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"builder-util": patch
3+
---
4+
5+
chore: removing unused `notice` LogLevel

‎packages/app-builder-lib/src/node-module-collector/index.ts‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import { TmpDir } from "temp-file"
33
import { NpmNodeModulesCollector } from "./npmNodeModulesCollector"
44
import { detectPackageManager, getPackageManagerCommand, PM } from "./packageManager"
55
import { PnpmNodeModulesCollector } from "./pnpmNodeModulesCollector"
6-
import { NodeModuleInfo } from "./types"
76
import { YarnBerryNodeModulesCollector } from "./yarnBerryNodeModulesCollector"
87
import { YarnNodeModulesCollector } from "./yarnNodeModulesCollector"
98
import { BunNodeModulesCollector } from "./bunNodeModulesCollector"
@@ -32,22 +31,6 @@ export function getCollectorByPackageManager(pm: PM, rootDir: string, tempDirMan
3231
}
3332
}
3433

35-
export function getNodeModules(
36-
pm: PM,
37-
{
38-
rootDir,
39-
tempDirManager,
40-
packageName,
41-
}: {
42-
rootDir: string
43-
tempDirManager: TmpDir
44-
packageName: string
45-
}
46-
): Promise<NodeModuleInfo[]> {
47-
const collector = getCollectorByPackageManager(pm, rootDir, tempDirManager)
48-
return collector.getNodeModules({ packageName })
49-
}
50-
5134
export const determinePackageManagerEnv = ({ projectDir, appDir, workspaceRoot }: { projectDir: string; appDir: string; workspaceRoot: string | Nullish }) =>
5235
new Lazy(async () => {
5336
const availableDirs = [workspaceRoot, projectDir, appDir].filter((it): it is string => !isEmptyOrSpaces(it))

‎packages/app-builder-lib/src/node-module-collector/moduleManager.ts‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,26 @@
1-
import { exists, isEmptyOrSpaces, log } from "builder-util"
1+
import { exists, isEmptyOrSpaces, log, LogLevel } from "builder-util"
22
import { PackageJson } from "./types"
33
import * as fs from "fs-extra"
44
import * as path from "path"
55
import * as semver from "semver"
66

7+
export enum LogMessageByKey {
8+
PKG_DUPLICATE_REF = "duplicate dependency references",
9+
PKG_NOT_FOUND = "cannot find path for dependency",
10+
PKG_NOT_ON_DISK = "dependency not found on disk",
11+
PKG_SELF_REF = "self-referential dependencies",
12+
PKG_OPTIONAL_NOT_INSTALLED = "missing optional dependencies",
13+
PKG_COLLECTOR_OUTPUT = "collector stderr output",
14+
}
15+
export const logMessageLevelByKey: Record<LogMessageByKey, LogLevel> = {
16+
[LogMessageByKey.PKG_DUPLICATE_REF]: "info",
17+
[LogMessageByKey.PKG_NOT_FOUND]: "warn",
18+
[LogMessageByKey.PKG_NOT_ON_DISK]: "warn",
19+
[LogMessageByKey.PKG_SELF_REF]: "debug",
20+
[LogMessageByKey.PKG_OPTIONAL_NOT_INSTALLED]: "info",
21+
[LogMessageByKey.PKG_COLLECTOR_OUTPUT]: "warn",
22+
}
23+
724
export type Package = { packageDir: string; packageJson: PackageJson }
825

926
// Type aliases for clarity
@@ -12,6 +29,7 @@ type RealPathCache = Record<string, Promise<string>>
1229
type ExistsCache = Record<string, Promise<boolean>>
1330
type LstatCache = Record<string, Promise<fs.Stats | null>>
1431
type PackageCache = Record<string, Promise<Package | null>>
32+
type LogSummaryCache = Record<LogMessageByKey, string[]>
1533

1634
export class ModuleManager {
1735
/** Cache for package.json contents (readJson) */
@@ -24,14 +42,19 @@ export class ModuleManager {
2442
readonly lstat: LstatCache
2543
/** Cache for package lookups (key: "packageName||fromDir||semverRange"). Use helper function `versionedCacheKey` */
2644
readonly packageData: PackageCache
45+
/** For logging purposes, just track all dependencies for each key */
46+
readonly logSummary: LogSummaryCache
2747

2848
private readonly jsonMap: Map<string, PackageJson | null> = new Map()
2949
private readonly realPathMap: Map<string, string> = new Map()
3050
private readonly existsMap: Map<string, boolean> = new Map()
3151
private readonly lstatMap: Map<string, fs.Stats | null> = new Map()
3252
private readonly packageDataMap: Map<string, Package | null> = new Map()
53+
private readonly logSummaryMap: Map<LogMessageByKey, string[]> = new Map()
3354

3455
constructor() {
56+
this.logSummary = this.createLogSummarySyncProxy()
57+
3558
this.exists = this.createAsyncProxy(this.existsMap, (p: string) => exists(p))
3659
this.json = this.createAsyncProxy(this.jsonMap, (p: string) => fs.readJson(p).catch(() => null))
3760
this.lstat = this.createAsyncProxy(this.lstatMap, (p: string) => fs.lstat(p).catch(() => null))
@@ -43,6 +66,37 @@ export class ModuleManager {
4366
})
4467
}
4568

69+
private createLogSummarySyncProxy(): LogSummaryCache {
70+
return new Proxy({} as LogSummaryCache, {
71+
get: (_, key: LogMessageByKey) => {
72+
if (!this.logSummaryMap.has(key)) {
73+
this.logSummaryMap.set(key, [])
74+
}
75+
return this.logSummaryMap.get(key)!
76+
},
77+
set: (_, key: LogMessageByKey, value: string[]) => {
78+
this.logSummaryMap.set(key, value)
79+
return true
80+
},
81+
has: (_, key: LogMessageByKey) => {
82+
return this.logSummaryMap.has(key)
83+
},
84+
// Add these to make Object.entries() work
85+
ownKeys: _ => {
86+
return Array.from(this.logSummaryMap.keys())
87+
},
88+
getOwnPropertyDescriptor: (_, key) => {
89+
if (this.logSummaryMap.has(key as LogMessageByKey)) {
90+
return {
91+
enumerable: true,
92+
configurable: true,
93+
}
94+
}
95+
return undefined
96+
},
97+
})
98+
}
99+
46100
// this allows dot-notation access while still supporting async retrieval
47101
// e.g., cache.packageJson[somePath] returns Promise<PackageJson>
48102
private createAsyncProxy<T>(map: Map<string, T>, compute: (key: string) => T | Promise<T>): Record<string, Promise<T>> {

‎packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { createWriteStream } from "fs-extra"
55
import { Lazy } from "lazy-val"
66
import * as path from "path"
77
import { hoist, type HoisterResult, type HoisterTree } from "./hoist"
8-
import { ModuleManager } from "./moduleManager"
8+
import { LogMessageByKey, ModuleManager } from "./moduleManager"
99
import { getPackageManagerCommand, PM } from "./packageManager"
1010
import type { Dependency, DependencyGraph, NodeModuleInfo, PackageJson } from "./types"
1111

@@ -46,12 +46,11 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
4646
* 4. Building a production dependency graph
4747
* 5. Hoisting the dependencies to their final locations
4848
* 6. Resolving and returning module information
49-
*
50-
* @param options - Configuration object
51-
* @param options.packageName - The name of the package to collect modules for
52-
* @returns Promise resolving to an array of NodeModuleInfo objects representing all collected modules
5349
*/
54-
public async getNodeModules({ packageName }: { packageName: string }): Promise<NodeModuleInfo[]> {
50+
public async getNodeModules({ packageName }: { packageName: string }): Promise<{
51+
nodeModules: NodeModuleInfo[]
52+
logSummary: ModuleManager["logSummary"]
53+
}> {
5554
const tree: ProdDepType = await this.getDependenciesTree(this.installOptions.manager)
5655

5756
await this.collectAllDependencies(tree, packageName)
@@ -63,9 +62,10 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
6362
})
6463

6564
await this._getNodeModules(hoisterResult.dependencies, this.nodeModules)
65+
6666
log.debug({ packageName, depCount: this.nodeModules.length }, "node modules collection complete")
6767

68-
return this.nodeModules
68+
return { nodeModules: this.nodeModules, logSummary: this.cache.logSummary }
6969
}
7070

7171
public abstract readonly installOptions: {
@@ -83,10 +83,6 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
8383
* Executes the appropriate package manager command to fetch the dependency tree and writes
8484
* the output to a temporary file. Includes retry logic to handle transient failures such as
8585
* incomplete JSON output or missing files. Will retry up to 1 time with exponential backoff.
86-
*
87-
* @param pm - The package manager to use (npm, yarn, pnpm, etc.)
88-
* @returns Promise resolving to the parsed dependency tree
89-
* @throws {Error} If the dependency tree cannot be retrieved after retries
9086
*/
9187
protected async getDependenciesTree(pm: PM): Promise<ProdDepType> {
9288
const command = getPackageManagerCommand(pm)
@@ -288,16 +284,17 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
288284

289285
for (const d of dependencies.values()) {
290286
const reference = [...d.references][0]
291-
const p = this.allDependencies.get(`${d.name}@${reference}`)?.path
287+
const key = `${d.name}@${reference}`
288+
const p = this.allDependencies.get(key)?.path
292289
if (p === undefined) {
293-
log.warn({ name: d.name, reference }, "cannot find path for dependency")
290+
this.cache.logSummary[LogMessageByKey.PKG_NOT_FOUND].push(key)
294291
continue
295292
}
296293

297294
// fix npm list issue
298295
// https://github.com/npm/cli/issues/8535
299296
if (!(await this.cache.exists[p])) {
300-
log.debug({ name: d.name, reference, p }, "dependency path does not exist")
297+
this.cache.logSummary[LogMessageByKey.PKG_NOT_ON_DISK].push(key)
301298
continue
302299
}
303300

@@ -388,6 +385,7 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
388385
}
389386
if (stderr.length > 0) {
390387
log.debug({ stderr }, "note: there was node module collector output on stderr")
388+
this.cache.logSummary[LogMessageByKey.PKG_COLLECTOR_OUTPUT].push(stderr)
391389
}
392390
const shouldResolve = code === 0 || shouldIgnore
393391
return shouldResolve ? resolve() : reject(new Error(`Node module collector process exited with code ${code}:\n${stderr}`))

‎packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { LogMessageByKey } from "./moduleManager.js"
12
import { NodeModulesCollector } from "./nodeModulesCollector.js"
23
import { PM } from "./packageManager.js"
34
import { NpmDependency } from "./types.js"
@@ -24,6 +25,7 @@ export class NpmNodeModulesCollector extends NodeModulesCollector<NpmDependency,
2425
if (!this.allDependencies.has(childDependencyId)) {
2526
this.allDependencies.set(childDependencyId, pkgOverride)
2627
}
28+
this.cache.logSummary[LogMessageByKey.PKG_DUPLICATE_REF].push(childDependencyId)
2729
continue
2830
}
2931

‎packages/app-builder-lib/src/node-module-collector/traversalNodeModulesCollector.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { log } from "builder-util"
2+
import * as path from "path"
3+
import { LogMessageByKey } from "./moduleManager"
24
import { NodeModulesCollector } from "./nodeModulesCollector"
35
import { PM } from "./packageManager.js"
46
import { TraversedDependency } from "./types.js"
5-
import * as path from "path"
67

78
// manual traversal of node_modules for package managers without CLI support for dependency tree extraction (e.g., bun) OR as a fallback (e.g. corepack enabled w/ strict mode)
89
export class TraversalNodeModulesCollector extends NodeModulesCollector<TraversedDependency, TraversedDependency> {
@@ -103,7 +104,7 @@ export class TraversalNodeModulesCollector extends NodeModulesCollector<Traverse
103104

104105
// Skip if this dependency resolves to the base directory or any parent we're already processing
105106
if (pkg.packageDir === resolvedPackageDir || pkg.packageDir === resolvedBaseDir) {
106-
log.debug({ ...logFields, resolvedPath: pkg.packageDir }, "skipping self-referential dependency")
107+
this.cache.logSummary[LogMessageByKey.PKG_SELF_REF].push(`${depName}@${depVersion}`)
107108
continue
108109
}
109110

@@ -118,8 +119,9 @@ export class TraversalNodeModulesCollector extends NodeModulesCollector<Traverse
118119
throw new Error(`Production dependency ${depName} not found for package ${moduleName}`)
119120
})
120121

121-
const optionalDeps = await buildPackage(pkg.optionalDependencies, (depName: string) => {
122+
const optionalDeps = await buildPackage(pkg.optionalDependencies, (depName: string, version: string) => {
122123
log.debug({ parent: moduleName, dependency: depName }, "optional dependency not installed, skipping")
124+
this.cache.logSummary[LogMessageByKey.PKG_OPTIONAL_NOT_INSTALLED].push(`${depName}@${version}`)
123125
})
124126

125127
return {

‎packages/app-builder-lib/src/util/appFileCopier.ts‎

Lines changed: 43 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import { isLibOrExe } from "../asar/unpackDetector"
88
import { Platform } from "../core"
99
import { excludedExts, FileMatcher } from "../fileMatcher"
1010
import { createElectronCompilerHost, NODE_MODULES_PATTERN } from "../fileTransformer"
11+
import { getCollectorByPackageManager, PM } from "../node-module-collector"
12+
import { LogMessageByKey, logMessageLevelByKey, ModuleManager } from "../node-module-collector/moduleManager"
1113
import { Packager } from "../packager"
1214
import { PlatformPackager } from "../platformPackager"
1315
import { AppFileWalker } from "./AppFileWalker"
1416
import { NodeModuleCopyHelper } from "./NodeModuleCopyHelper"
1517
import { NodeModuleInfo } from "./packageDependencies"
16-
import { getNodeModules, PM } from "../node-module-collector"
1718

1819
const BOWER_COMPONENTS_PATTERN = `${path.sep}bower_components${path.sep}`
1920
/** @internal */
@@ -178,34 +179,7 @@ function validateFileSet(fileSet: ResolvedFileSet): ResolvedFileSet {
178179

179180
/** @internal */
180181
export async function computeNodeModuleFileSets(platformPackager: PlatformPackager<any>, mainMatcher: FileMatcher): Promise<Array<ResolvedFileSet>> {
181-
const packager = platformPackager.info
182-
const { tempDirManager, cancellationToken, appDir, projectDir } = packager
183-
184-
let deps: Array<NodeModuleInfo> = []
185-
const searchDirectories = Array.from(new Set([appDir, projectDir, await packager.getWorkspaceRoot()])).filter((it): it is string => isEmptyOrSpaces(it) === false)
186-
const pmApproaches = [await packager.getPackageManager(), PM.TRAVERSAL]
187-
for (const pm of pmApproaches) {
188-
for (const dir of searchDirectories) {
189-
log.info({ pm, searchDir: dir }, "searching for node modules")
190-
const options = { rootDir: dir, tempDirManager, cancellationToken, packageName: packager.metadata.name! }
191-
deps = await getNodeModules(pm, options)
192-
if (deps.length > 0) {
193-
break
194-
}
195-
const attempt = searchDirectories.indexOf(dir)
196-
if (attempt < searchDirectories.length - 1) {
197-
log.info({ searchDir: dir, attempt }, "no node modules found in collection, trying next search directory")
198-
}
199-
}
200-
if (deps.length > 0) {
201-
log.debug({ pm, nodeModules: deps }, "collected node modules")
202-
break
203-
}
204-
}
205-
if (deps.length === 0) {
206-
log.warn({ searchDirectories: searchDirectories.map(it => log.filePath(it)) }, "no node modules returned while searching directories")
207-
return []
208-
}
182+
const deps = await collectNodeModulesWithLogging(platformPackager)
209183

210184
const nodeModuleExcludedExts = getNodeModuleExcludedExts(platformPackager)
211185
// serial execution because copyNodeModules is concurrent and so, no need to increase queue/pressure
@@ -236,6 +210,46 @@ export async function computeNodeModuleFileSets(platformPackager: PlatformPackag
236210
return result
237211
}
238212

213+
async function collectNodeModulesWithLogging(platformPackager: PlatformPackager<any>) {
214+
const packager = platformPackager.info
215+
const { tempDirManager, appDir, projectDir } = packager
216+
217+
let deps: { nodeModules: NodeModuleInfo[]; logSummary: ModuleManager["logSummary"] } | undefined = undefined
218+
219+
const searchDirectories = Array.from(new Set([appDir, projectDir, await packager.getWorkspaceRoot()])).filter((it): it is string => isEmptyOrSpaces(it) === false)
220+
const pmApproaches = [await packager.getPackageManager(), PM.TRAVERSAL]
221+
for (const pm of pmApproaches) {
222+
for (const dir of searchDirectories) {
223+
log.info({ pm, searchDir: dir }, "searching for node modules")
224+
const collector = getCollectorByPackageManager(pm, dir, tempDirManager)
225+
deps = await collector.getNodeModules({ packageName: packager.metadata.name! })
226+
if (deps.nodeModules.length > 0) {
227+
break
228+
}
229+
const attempt = searchDirectories.indexOf(dir)
230+
if (attempt < searchDirectories.length - 1) {
231+
log.info({ searchDir: dir, attempt }, "no node modules found in collection, trying next search directory")
232+
}
233+
}
234+
if (deps?.nodeModules?.length) {
235+
log.debug({ pm, nodeModules: deps.nodeModules }, "collected node modules")
236+
break
237+
}
238+
}
239+
if (!deps?.nodeModules?.length) {
240+
log.warn({ searchDirectories: searchDirectories.map(it => log.filePath(it)) }, "no node modules returned while searching directories")
241+
return []
242+
}
243+
244+
const summary = Object.entries(deps.logSummary ?? {}).filter(([, dependencies]) => Array.isArray(dependencies) && dependencies.length > 0)
245+
for (const [errorMessage, dependencies] of summary) {
246+
const logLevel = logMessageLevelByKey[errorMessage as LogMessageByKey] || "debug"
247+
log[logLevel]({ dependencies }, errorMessage)
248+
}
249+
250+
return deps.nodeModules
251+
}
252+
239253
async function compileUsingElectronCompile(mainFileSet: ResolvedFileSet, packager: Packager): Promise<ResolvedFileSet> {
240254
log.info("compiling using electron-compile")
241255

‎packages/builder-util/src/log.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ export function setPrinter(value: ((message: string) => void) | null) {
1515
printer = value
1616
}
1717

18-
export type LogLevel = "info" | "warn" | "debug" | "notice" | "error"
18+
export type LogLevel = "info" | "warn" | "debug" | "error"
1919

2020
export const PADDING = 2
2121

0 commit comments

Comments
 (0)