feat: use cacheKey in fetch-remote-file

Author: wardpeet

packages/gatsby-source-contentful/src/__tests__/gatsby-plugin-image.js

@@ -4,7 +4,7 @@ import _ from "lodash"
 import nock from "nock"
 import path from "path"
 import { generateImageSource, getBase64Image } from "../gatsby-plugin-image"
-import * as coreUtils from "gatsby-core-utils"
+import * as coreUtils from "gatsby-core-utils/fetch-remote-file"
 
 nock.disableNetConnect()
 
@@ -99,6 +99,9 @@ describe(`contentful extend node type`, () => {
         title: `Contentful Logo PNG`,
         description: ``,
         node_locale: `en-US`,
+        internal: {
+          contentDigest: `123`,
+        },
       },
       options: {
         width: 200,

packages/gatsby-source-contentful/src/gatsby-plugin-image.js

@@ -1,6 +1,6 @@
 // @ts-check
 import fs from "fs-extra"
-import { fetchRemoteFile } from "gatsby-core-utils"
+import { fetchRemoteFile } from "gatsby-core-utils/fetch-remote-file"
 import path from "path"
 import {
   createUrl,
@@ -61,8 +61,9 @@ export const getBase64Image = (imageProps, cache) => {
 
     const absolutePath = await fetchRemoteFile({
       url: requestUrl,
-      cache,
+      directory: cache.directory,
       ext: extension,
+      cacheKey: imageProps.image.internal.contentDigest,
     })
 
     const base64 = (await fs.readFile(absolutePath)).toString(`base64`)
@@ -97,8 +98,9 @@ const getTracedSVG = async ({ image, options, cache }) => {
   const absolutePath = await fetchRemoteFile({
     url,
     name,
-    cache,
+    directory: cache.directory,
     ext: extension,
+    cacheKey: image.internal.contentDigest,
   })
 
   return traceSVG({
@@ -147,8 +149,9 @@ const getDominantColor = async ({ image, options, cache }) => {
     const absolutePath = await fetchRemoteFile({
       url,
       name,
-      cache,
+      directory: cache.directory,
       ext: extension,
+      cacheKey: image.internal.contentDigest,
     })
 
     if (!(`getDominantColor` in pluginSharp)) {

packages/gatsby-source-filesystem/src/__tests__/create-remote-file-node.js

@@ -22,7 +22,7 @@ jest.mock(`got`, () => {
   }
 })
 
-jest.mock(`gatsby-core-utils`, () => {
+jest.mock(`gatsby-core-utils/fetch-remote-file`, () => {
   return {
     fetchRemoteFile: jest.fn(),
   }
@@ -37,7 +37,7 @@ const reporter = {}
 
 const createRemoteFileNode = require(`../create-remote-file-node`)
 const { createFileNode } = require(`../create-file-node`)
-const { fetchRemoteFile } = require(`gatsby-core-utils`)
+const { fetchRemoteFile } = require(`gatsby-core-utils/fetch-remote-file`)
 
 const createMockCache = () => {
   return {

packages/gatsby-source-filesystem/src/create-remote-file-node.js

@@ -1,4 +1,4 @@
-const { fetchRemoteFile } = require(`gatsby-core-utils`)
+const { fetchRemoteFile } = require(`gatsby-core-utils/fetch-remote-file`)
 const { isWebUri } = require(`valid-url`)
 const { createFileNode } = require(`./create-file-node`)
 

packages/gatsby-source-shopify/src/resolve-gatsby-image-data.ts

@@ -1,4 +1,4 @@
-import { fetchRemoteFile } from "gatsby-core-utils"
+import { fetchRemoteFile } from "gatsby-core-utils/fetch-remote-file"
 import {
   generateImageData,
   getLowResolutionImageURL,
@@ -10,22 +10,26 @@ import {
 import { IGatsbyImageFieldArgs } from "gatsby-plugin-image/graphql-utils"
 import { readFileSync } from "fs"
 import { IShopifyImage, urlBuilder } from "./get-shopify-image"
+import type { Node } from "gatsby"
 
 type IImageWithPlaceholder = IImage & {
   placeholder: string
 }
 
 async function getImageBase64({
   imageAddress,
-  cache,
+  directory,
+  contentDigest,
 }: {
   imageAddress: string
-  cache: any
+  directory: string
+  contentDigest: string
 }): Promise<string> {
   // Downloads file to the site cache and returns the file path for the given image (this is a path on the host system, not a URL)
   const filePath = await fetchRemoteFile({
     url: imageAddress,
-    cache,
+    directory,
+    cacheKey: contentDigest,
   })
   const buffer = readFileSync(filePath)
   return buffer.toString(`base64`)
@@ -99,7 +103,8 @@ export function makeResolveGatsbyImageData(cache: any) {
       })
       const imageBase64 = await getImageBase64({
         imageAddress: lowResImageURL,
-        cache,
+        directory: cache.directory as string,
+        contentDigest: image.internal.contentDigest,
       })
 
       // This would be your own function to download and generate a low-resolution placeholder

packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js

@@ -8,7 +8,7 @@ const { isWebUri } = require(`valid-url`)
 const Queue = require(`better-queue`)
 const readChunk = require(`read-chunk`)
 const fileType = require(`file-type`)
-
+const { fetchRemoteFile } = require(`gatsby-core-utils/fetch-remote-file`)
 const { createFileNode } = require(`gatsby-source-filesystem/create-file-node`)
 const {
   getRemoteFileExtension,
@@ -142,94 +142,6 @@ async function pushToQueue(task, cb) {
 /******************
  * Core Functions *
  ******************/
-
-/**
- * requestRemoteNode
- * --
- * Download the requested file
- *
- * @param  {String}   url
- * @param  {Headers}  headers
- * @param  {String}   tmpFilename
- * @param  {Object}   httpOpts
- * @param  {number}   attempt
- * @return {Promise<Object>}  Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
- */
-const requestRemoteNode = (url, headers, tmpFilename, httpOpts, attempt = 1) =>
-  new Promise((resolve, reject) => {
-    let timeout
-
-    // Called if we stall without receiving any data
-    const handleTimeout = async () => {
-      fsWriteStream.close()
-      fs.removeSync(tmpFilename)
-      if (attempt < STALL_RETRY_LIMIT) {
-        // Retry by calling ourself recursively
-        resolve(
-          requestRemoteNode(url, headers, tmpFilename, httpOpts, attempt + 1)
-        )
-      } else {
-        processingCache[url] = null
-        totalJobs -= 1
-        bar.total = totalJobs
-        reject(
-          new Error(
-            `Failed to download ${url} after ${STALL_RETRY_LIMIT} attempts`
-          )
-        )
-      }
-    }
-
-    const resetTimeout = () => {
-      if (timeout) {
-        clearTimeout(timeout)
-      }
-      timeout = setTimeout(handleTimeout, STALL_TIMEOUT)
-    }
-
-    const responseStream = got.stream(url, {
-      headers,
-      timeout: { send: CONNECTION_TIMEOUT },
-      ...httpOpts,
-    })
-    const fsWriteStream = fs.createWriteStream(tmpFilename)
-    responseStream.pipe(fsWriteStream)
-
-    // If there's a 400/500 response or other error.
-    responseStream.on(`error`, error => {
-      if (timeout) {
-        clearTimeout(timeout)
-      }
-      processingCache[url] = null
-      totalJobs -= 1
-      bar.total = totalJobs
-      fs.removeSync(tmpFilename)
-      console.error(error)
-      reject(error)
-    })
-
-    fsWriteStream.on(`error`, error => {
-      if (timeout) {
-        clearTimeout(timeout)
-      }
-      processingCache[url] = null
-      totalJobs -= 1
-      bar.total = totalJobs
-      reject(error)
-    })
-
-    responseStream.on(`response`, response => {
-      resetTimeout()
-
-      fsWriteStream.on(`finish`, () => {
-        if (timeout) {
-          clearTimeout(timeout)
-        }
-        resolve(response)
-      })
-    })
-  })
-
 /**
  * processRemoteNode
  * --
@@ -249,71 +161,14 @@ async function processRemoteNode({
   ext,
   name,
 }) {
-  const pluginCacheDir = cache.directory
-  // See if there's response headers for this url
-  // from a previous request.
-  const cachedHeaders = await cache.get(cacheId(url))
-
-  const headers = { ...httpHeaders }
-  if (cachedHeaders && cachedHeaders.etag) {
-    headers[`If-None-Match`] = cachedHeaders.etag
-  }
-
-  // Add htaccess authentication if passed in. This isn't particularly
-  // extensible. We should define a proper API that we validate.
-  const httpOpts = {}
-  if (auth?.htaccess_pass && auth?.htaccess_user) {
-    headers[`Authorization`] = `Basic ${btoa(
-      `${auth.htaccess_user}:${auth.htaccess_pass}`
-    )}`
-  }
-
-  // Create the temp and permanent file names for the url.
-  const digest = createContentDigest(url)
-  if (!name) {
-    name = getRemoteFileName(url)
-  }
-  if (!ext) {
-    ext = getRemoteFileExtension(url)
-  }
-
-  const tmpFilename = createFilePath(pluginCacheDir, `tmp-${digest}`, ext)
-
-  // Fetch the file.
-  const response = await requestRemoteNode(url, headers, tmpFilename, httpOpts)
-
-  if (response.statusCode == 200) {
-    // Save the response headers for future requests.
-    await cache.set(cacheId(url), response.headers)
-  }
-
-  // If the user did not provide an extension and we couldn't get one from remote file, try and guess one
-  if (ext === ``) {
-    const buffer = readChunk.sync(tmpFilename, 0, fileType.minimumBytes)
-    const filetype = fileType(buffer)
-    if (filetype) {
-      ext = `.${filetype.ext}`
-    }
-  }
-
-  const filename = createFilePath(
-    path.join(pluginCacheDir, digest),
-    String(name),
-    ext
-  )
-
-  // If the status code is 200, move the piped temp file to the real name.
-  if (response.statusCode === 200) {
-    await fs.move(tmpFilename, filename, { overwrite: true })
-    // Else if 304, remove the empty response.
-  } else {
-    processingCache[url] = null
-    totalJobs -= 1
-
-    bar.total = totalJobs
-
-    await fs.remove(tmpFilename)
-  }
+  const filename = await fetchRemoteFile({
+    url,
+    httpHeaders,
+    auth,
+    ext,
+    name,
+    directory: cache.directory,
+  })
 
   // Create the file node.
   const fileNode = await createFileNode(filename, createNodeId, {})

packages/gatsby-transformer-sqip/src/extend-node-type.js

@@ -12,7 +12,7 @@ const {
   GraphQLBoolean,
 } = require(`gatsby/graphql`)
 const { queueImageResizing } = require(`gatsby-plugin-sharp`)
-const { fetchRemoteFile } = require(`gatsby-core-utils`)
+const { fetchRemoteFile } = require(`gatsby-core-utils/fetch-remote-file`)
 const {
   DuotoneGradientType,
   ImageCropFocusType,