chore: upgrade got package (#32928)

Co-authored-by: Lennart <lekoarts@gmail.com>

Author: wardpeet

package.json

@@ -14,7 +14,6 @@
     "@types/common-tags": "^1.8.0",
     "@types/express": "^4.17.3",
     "@types/fs-extra": "^9.0.12",
-    "@types/got": "^9.6.11",
     "@types/jaeger-client": "^3.18.0",
     "@types/jest": "^24.9.1",
     "@types/joi": "^14.3.4",

packages/gatsby-core-utils/package.json

@@ -34,6 +34,7 @@
     "configstore": "^5.0.1",
     "file-type": "^16.5.3",
     "fs-extra": "^10.0.0",
+    "got": "^11.8.2",
     "node-object-hash": "^2.3.8",
     "proper-lockfile": "^4.1.2",
     "tmp": "^0.2.1",
@@ -45,6 +46,7 @@
     "@types/ci-info": "2.0.0",
     "babel-preset-gatsby-package": "^1.14.0-next.0",
     "cross-env": "^7.0.3",
+    "msw": "^0.35.0",
     "typescript": "^4.3.5"
   },
   "engines": {

packages/gatsby-core-utils/src/__tests__/create-remote-file-node-integration.js

@@ -129,6 +129,8 @@ function createMockCache() {
     path.join(os.tmpdir(), `gatsby-source-filesystem-`)
   )
 
+  fs.ensureDir(tmpDir)
+
   return {
     get: jest.fn(),
     set: jest.fn(),

packages/gatsby-core-utils/src/fetch-remote-file.ts

@@ -1,7 +1,6 @@
-import got from "got"
+import got, { Headers, Options } from "got"
 import fileType from "file-type"
 import path from "path"
-import { IncomingMessage, OutgoingHttpHeaders } from "http"
 import fs from "fs-extra"
 import { createContentDigest } from "./create-content-digest"
 import {
@@ -10,7 +9,8 @@ import {
   createFilePath,
 } from "./filename-utils"
 
-import { GatsbyCache } from "gatsby"
+import type { IncomingMessage } from "http"
+import type { GatsbyCache } from "gatsby"
 
 export interface IFetchRemoteFileOptions {
   url: string
@@ -19,7 +19,7 @@ export interface IFetchRemoteFileOptions {
     htaccess_pass?: string
     htaccess_user?: string
   }
-  httpHeaders?: OutgoingHttpHeaders
+  httpHeaders?: Headers
   ext?: string
   name?: string
 }
@@ -57,10 +57,10 @@ const INCOMPLETE_RETRY_LIMIT = process.env.GATSBY_INCOMPLETE_RETRY_LIMIT
  * @return {Promise<Object>}  Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
  */
 const requestRemoteNode = (
-  url: got.GotUrl,
-  headers: OutgoingHttpHeaders,
+  url: string | URL,
+  headers: Headers,
   tmpFilename: string,
-  httpOptions: got.GotOptions<string | null> | undefined,
+  httpOptions?: Options,
   attempt: number = 1
 ): Promise<IncomingMessage> =>
   new Promise((resolve, reject) => {
@@ -71,12 +71,15 @@ const requestRemoteNode = (
     const handleTimeout = async (): Promise<void> => {
       fsWriteStream.close()
       fs.removeSync(tmpFilename)
+
       if (attempt < STALL_RETRY_LIMIT) {
         // Retry by calling ourself recursively
         resolve(
           requestRemoteNode(url, headers, tmpFilename, httpOptions, attempt + 1)
         )
       } else {
+        // TODO move to new Error type
+        // eslint-disable-next-line prefer-promise-reject-errors
         reject(`Failed to download ${url} after ${STALL_RETRY_LIMIT} attempts`)
       }
     }
@@ -93,15 +96,21 @@ const requestRemoteNode = (
         send: CONNECTION_TIMEOUT, // https://github.com/sindresorhus/got#timeout
       },
       ...httpOptions,
+      isStream: true,
     })
 
     let haveAllBytesBeenWritten = false
+    // Fixes a bug in latest got where progress.total gets reset when stream ends, even if it wasn't complete.
+    let totalSize: number | null = null
     responseStream.on(`downloadProgress`, progress => {
       if (
-        progress.transferred === progress.total ||
-        progress.total === null ||
-        progress.total === undefined
+        progress.total != null &&
+        (!totalSize || totalSize < progress.total)
       ) {
+        totalSize = progress.total
+      }
+
+      if (progress.transferred === totalSize || totalSize === null) {
         haveAllBytesBeenWritten = true
       }
     })
@@ -113,29 +122,36 @@ const requestRemoteNode = (
       if (timeout) {
         clearTimeout(timeout)
       }
+
+      fsWriteStream.close()
       fs.removeSync(tmpFilename)
       reject(error)
     })
 
-    fsWriteStream.on(`error`, (error: any) => {
+    fsWriteStream.on(`error`, (error: unknown) => {
       if (timeout) {
         clearTimeout(timeout)
       }
+
       reject(error)
     })
 
     responseStream.on(`response`, response => {
       resetTimeout()
 
       fsWriteStream.on(`finish`, () => {
+        if (timeout) {
+          clearTimeout(timeout)
+        }
+
         fsWriteStream.close()
 
         // We have an incomplete download
         if (!haveAllBytesBeenWritten) {
           fs.removeSync(tmpFilename)
 
           if (attempt < INCOMPLETE_RETRY_LIMIT) {
-            resolve(
+            return resolve(
               requestRemoteNode(
                 url,
                 headers,
@@ -145,16 +161,15 @@ const requestRemoteNode = (
               )
             )
           } else {
-            reject(
+            // TODO move to new Error type
+            // eslint-disable-next-line prefer-promise-reject-errors
+            return reject(
               `Failed to download ${url} after ${INCOMPLETE_RETRY_LIMIT} attempts`
             )
           }
         }
 
-        if (timeout) {
-          clearTimeout(timeout)
-        }
-        resolve(response)
+        return resolve(response)
       })
     })
   })
@@ -177,14 +192,11 @@ export async function fetchRemoteFile({
     headers[`If-None-Match`] = cachedHeaders.etag
   }
 
-  // Add Basic authentication if passed in:
-  // https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#username
-  // The "auth" API isn't particularly extensible, we should define an API that we validate
-  const httpOptions: got.GotOptions<string | null> = {}
+  // Add htaccess authentication if passed in. This isn't particularly
+  // extensible. We should define a proper API that we validate.
+  const httpOptions: Options = {}
   if (auth && (auth.htaccess_pass || auth.htaccess_user)) {
-    // @ts-ignore - We use outdated @types/got typings. Once we update got everywhere we can remove @types/got and have correct typings
     httpOptions.username = auth.htaccess_user
-    // @ts-ignore - see above
     httpOptions.password = auth.htaccess_pass
   }
 

packages/gatsby-dev-cli/package.json

@@ -17,7 +17,7 @@
     "execa": "^5.1.1",
     "find-yarn-workspace-root": "^2.0.0",
     "fs-extra": "^10.0.0",
-    "got": "^10.7.0",
+    "got": "^11.8.2",
     "is-absolute": "^1.0.0",
     "lodash": "^4.17.21",
     "signal-exit": "^3.0.3",

packages/gatsby-plugin-sharp/package.json

@@ -15,7 +15,7 @@
     "gatsby-core-utils": "^2.14.0-next.0",
     "gatsby-plugin-utils": "^1.14.0-next.0",
     "gatsby-telemetry": "^2.14.0-next.0",
-    "got": "^10.7.0",
+    "got": "^11.8.2",
     "lodash": "^4.17.21",
     "mini-svg-data-uri": "^1.3.3",
     "potrace": "^2.1.8",

packages/gatsby-source-filesystem/package.json

@@ -25,8 +25,7 @@
     "@babel/cli": "^7.14.8",
     "@babel/core": "^7.14.8",
     "babel-preset-gatsby-package": "^1.14.0-next.0",
-    "cross-env": "^7.0.3",
-    "msw": "^0.33.2"
+    "cross-env": "^7.0.3"
   },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem#readme",
   "keywords": [