From af4b3a95ddafc6f4f4c1f4a2ad20b8442ccc9887 Mon Sep 17 00:00:00 2001 From: StefanStojanovic Date: Fri, 26 May 2023 12:32:06 +0200 Subject: [PATCH] win,install: only download target_arch node.lib Instead of downloading node.lib for all architectures, just download the one that will be needed. Install.js changed to enable downloading just node.lib for node versions that already have tarball downloaded and extracted. Not fetching lib now fails the installation. Increased installVersion because of the changes. Refs: https://github.com/nodejs/node-gyp/issues/2847 --- lib/install.js | 169 +++++++++++++++++++++++------------------- package.json | 2 +- test/test-download.js | 2 +- 3 files changed, 95 insertions(+), 78 deletions(-) diff --git a/lib/install.js b/lib/install.js index 2d5db4e97d..1eb9f14c67 100644 --- a/lib/install.js +++ b/lib/install.js @@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline) async function install (fs, gyp, argv) { const release = processRelease(argv, gyp, process.version, process.release) + // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only. + const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : '' + // Used to prevent downloading tarball if only new node.lib is required on Windows. + let shouldDownloadTarball = true // Determine which node dev files version we are installing log.verbose('install', 'input version string %j', release.version) @@ -92,6 +96,26 @@ async function install (fs, gyp, argv) { } } log.verbose('install', 'version is good') + if (win) { + log.verbose('on Windows; need to check node.lib') + const nodeLibPath = path.resolve(devDir, arch, 'node.lib') + try { + await fs.promises.stat(nodeLibPath) + } catch (err) { + if (err.code === 'ENOENT') { + log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version) + try { + shouldDownloadTarball = false + return await go() + } catch (err) { + return rollback(err) + } + } else if (err.code === 'EACCES') { + return eaccesFallback(err) + } + throw err + } + } } else { try { return await go() @@ -179,6 +203,7 @@ async function install (fs, gyp, argv) { } // download the tarball and extract! + // Ommited on Windows if only new node.lib is required // on Windows there can be file errors from tar if parallel installs // are happening (not uncommon with multiple native modules) so @@ -186,59 +211,61 @@ async function install (fs, gyp, argv) { const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir try { - if (tarPath) { - await tar.extract({ - file: tarPath, - strip: 1, - filter: isValid, - onwarn, - cwd: tarExtractDir - }) - } else { - try { - const res = await download(gyp, release.tarballUrl) + if (shouldDownloadTarball) { + if (tarPath) { + await tar.extract({ + file: tarPath, + strip: 1, + filter: isValid, + onwarn, + cwd: tarExtractDir + }) + } else { + try { + const res = await download(gyp, release.tarballUrl) - if (res.status !== 200) { - throw new Error(`${res.status} response downloading ${release.tarballUrl}`) - } + if (res.status !== 200) { + throw new Error(`${res.status} response downloading ${release.tarballUrl}`) + } - await streamPipeline( - res.body, - // content checksum - new ShaSum((_, checksum) => { - const filename = path.basename(release.tarballUrl).trim() - contentShasums[filename] = checksum - log.verbose('content checksum', filename, checksum) - }), - tar.extract({ - strip: 1, - cwd: tarExtractDir, - filter: isValid, - onwarn - }) - ) - } catch (err) { + await streamPipeline( + res.body, + // content checksum + new ShaSum((_, checksum) => { + const filename = path.basename(release.tarballUrl).trim() + contentShasums[filename] = checksum + log.verbose('content checksum', filename, checksum) + }), + tar.extract({ + strip: 1, + cwd: tarExtractDir, + filter: isValid, + onwarn + }) + ) + } catch (err) { // something went wrong downloading the tarball? - if (err.code === 'ENOTFOUND') { - throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' + + if (err.code === 'ENOTFOUND') { + throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' + 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' + 'network settings.') + } + throw err } - throw err } - } - // invoked after the tarball has finished being extracted - if (extractErrors || extractCount === 0) { - throw new Error('There was a fatal problem while downloading/extracting the tarball') - } + // invoked after the tarball has finished being extracted + if (extractErrors || extractCount === 0) { + throw new Error('There was a fatal problem while downloading/extracting the tarball') + } - log.verbose('tarball', 'done parsing tarball') + log.verbose('tarball', 'done parsing tarball') + } const installVersionPath = path.resolve(tarExtractDir, 'installVersion') await Promise.all([ - // need to download node.lib - ...(win ? downloadNodeLib() : []), + // need to download node.lib + ...(win ? [downloadNodeLib()] : []), // write the "installVersion" file fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'), // Only download SHASUMS.txt if we downloaded something in need of SHA verification @@ -293,43 +320,33 @@ async function install (fs, gyp, argv) { log.verbose('checksum data', JSON.stringify(expectShasums)) } - function downloadNodeLib () { + async function downloadNodeLib () { log.verbose('on Windows; need to download `' + release.name + '.lib`...') - const archs = ['ia32', 'x64', 'arm64'] - return archs.map(async (arch) => { - const dir = path.resolve(tarExtractDir, arch) - const targetLibPath = path.resolve(dir, release.name + '.lib') - const { libUrl, libPath } = release[arch] - const name = `${arch} ${release.name}.lib` - log.verbose(name, 'dir', dir) - log.verbose(name, 'url', libUrl) - - await fs.promises.mkdir(dir, { recursive: true }) - log.verbose('streaming', name, 'to:', targetLibPath) - - const res = await download(gyp, libUrl) - - if (res.status === 403 || res.status === 404) { - if (arch === 'arm64') { - // Arm64 is a newer platform on Windows and not all node distributions provide it. - log.verbose(`${name} was not found in ${libUrl}`) - } else { - log.warn(`${name} was not found in ${libUrl}`) - } - return - } else if (res.status !== 200) { - throw new Error(`${res.status} status code downloading ${name}`) - } + const dir = path.resolve(tarExtractDir, arch) + const targetLibPath = path.resolve(dir, release.name + '.lib') + const { libUrl, libPath } = release[arch] + const name = `${arch} ${release.name}.lib` + log.verbose(name, 'dir', dir) + log.verbose(name, 'url', libUrl) + + await fs.promises.mkdir(dir, { recursive: true }) + log.verbose('streaming', name, 'to:', targetLibPath) + + const res = await download(gyp, libUrl) + + // Since only required node.lib is downloaded throw error if it is not fetched + if (res.status !== 200) { + throw new Error(`${res.status} status code downloading ${name}`) + } - return streamPipeline( - res.body, - new ShaSum((_, checksum) => { - contentShasums[libPath] = checksum - log.verbose('content checksum', libPath, checksum) - }), - fs.createWriteStream(targetLibPath) - ) - }) + return streamPipeline( + res.body, + new ShaSum((_, checksum) => { + contentShasums[libPath] = checksum + log.verbose('content checksum', libPath, checksum) + }), + fs.createWriteStream(targetLibPath) + ) } // downloadNodeLib() } // go() diff --git a/package.json b/package.json index 15cb6a72ea..331599447f 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "gyp" ], "version": "9.3.1", - "installVersion": 10, + "installVersion": 11, "author": "Nathan Rajlich (http://tootallnate.net)", "repository": { "type": "git", diff --git a/test/test-download.js b/test/test-download.js index 6eeba8a1dd..1dd5a51b06 100644 --- a/test/test-download.js +++ b/test/test-download.js @@ -180,7 +180,7 @@ describe('download', function () { await util.promisify(install)(prog, []) const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8') - assert.strictEqual(data, '10\n', 'correct installVersion') + assert.strictEqual(data, '11\n', 'correct installVersion') const list = await fs.promises.readdir(path.join(expectedDir, 'include/node')) assert.ok(list.includes('common.gypi'))