feat(gatsby-core-utils): Add support httpAgent

packages/gatsby-core-utils/src/fetch-remote-file.ts

@@ -1,7 +1,7 @@
 import got from "got"
 import fileType from "file-type"
 import path from "path"
-import { IncomingMessage, OutgoingHttpHeaders } from "http"
+import { IncomingMessage, OutgoingHttpHeaders, Agent } from "http"
 import fs from "fs-extra"
 import { createContentDigest } from "./create-content-digest"
 import {
@@ -18,7 +18,14 @@ export interface IFetchRemoteFileOptions {
   auth?: {
     htaccess_pass?: string
     htaccess_user?: string
-  }
+  },
+  httpOptions?: {
+    auth?: string
+    agent?: {
+      http?: Agent
+      https?: Agent
+    }
+  },
   httpHeaders?: OutgoingHttpHeaders
   ext?: string
   name?: string
@@ -60,7 +67,7 @@ const requestRemoteNode = (
   url: got.GotUrl,
   headers: OutgoingHttpHeaders,
   tmpFilename: string,
-  httpOpts: got.GotOptions<string | null> | undefined,
+  httpOpts: got.GotOptions | undefined,
   attempt: number = 1
 ): Promise<IncomingMessage> =>
   new Promise((resolve, reject) => {
@@ -163,6 +170,7 @@ export async function fetchRemoteFile({
   url,
   cache,
   auth = {},
+  httpOptions = {},
   httpHeaders = {},
   ext,
   name,
@@ -179,9 +187,8 @@ export async function fetchRemoteFile({
 
   // Add htaccess authentication if passed in. This isn't particularly
   // extensible. We should define a proper API that we validate.
-  const httpOpts: got.GotOptions<string | null> = {}
   if (auth && (auth.htaccess_pass || auth.htaccess_user)) {
-    httpOpts.auth = `${auth.htaccess_user}:${auth.htaccess_pass}`
+    httpOptions.auth = `${auth.htaccess_user}:${auth.htaccess_pass}`
   }
 
   // Create the temp and permanent file names for the url.
@@ -196,7 +203,7 @@ export async function fetchRemoteFile({
   const tmpFilename = createFilePath(pluginCacheDir, `tmp-${digest}`, ext)
 
   // Fetch the file.
-  const response = await requestRemoteNode(url, headers, tmpFilename, httpOpts)
+  const response = await requestRemoteNode(url, headers, tmpFilename, httpOptions)
 
   if (response.statusCode === 200) {
     // Save the response headers for future requests.

packages/gatsby-source-filesystem/README.md

@@ -183,6 +183,10 @@ createRemoteFileNode({
   // Adds htaccess authentication to the download request if passed in.
   auth: { htaccess_user: `USER`, htaccess_pass: `PASSWORD` },
 
+  // OPTIONAL
+  // Adds extra http options for the got module (npm) cf https://github.com/sindresorhus/got#agent
+  httpOptions: { agent: { http: http.Agent, https: https.Agent } },
+
   // OPTIONAL
   // Adds extra http headers to download request if passed in.
   httpHeaders: { Authorization: `Bearer someAccessToken` },

packages/gatsby-source-filesystem/index.d.ts

@@ -1,3 +1,4 @@
+import { Agent } from 'http'
 import { Node, Store, NodePluginArgs } from "gatsby"
 
 /**
@@ -37,7 +38,14 @@ export interface CreateRemoteFileNodeArgs {
   auth?: {
     htaccess_user: string
     htaccess_pass: string
-  }
+  },
+  httpOptions?: {
+    auth?: string,
+    agent?: {
+      http?: Agent,
+      https?: Agent
+    }
+  },
   httpHeaders?: object
   ext?: string
   name?: string

packages/gatsby-source-filesystem/src/create-remote-file-node.js

@@ -1,3 +1,4 @@
+const { Agent } = require('http');
 const fs = require(`fs-extra`)
 const { createContentDigest } = require(`gatsby-core-utils`)
 const path = require(`path`)
@@ -40,6 +41,10 @@ let showFlagWarning = !!process.env.GATSBY_EXPERIMENTAL_REMOTE_FILE_PLACEHOLDER
  * @param  {Function} options.createNode
  * @param  {Function} options.getCache
  * @param  {Auth} [options.auth]
+ * @param  {Object} [options.httpOptions]
+ * @param  {Object} [options.httpOptions.agent]
+ * @param  {Agent} [options.httpOptions.agent.http]
+ * @param  {Agent} [options.httpOptions.agent.https]
  * @param  {Reporter} [options.reporter]
  */
 
@@ -103,6 +108,7 @@ async function processRemoteNode({
   parentNodeId,
   auth = {},
   httpHeaders = {},
+  httpOptions = {},
   createNodeId,
   ext,
   name,
@@ -122,6 +128,7 @@ async function processRemoteNode({
       cache,
       auth,
       httpHeaders,
+      httpOptions,
       ext,
       name,
     })
@@ -201,6 +208,7 @@ module.exports = function createRemoteFileNode({
   getCache,
   parentNodeId = null,
   auth = {},
+  httpOptions = {},
   httpHeaders = {},
   createNodeId,
   ext = null,
@@ -260,6 +268,7 @@ module.exports = function createRemoteFileNode({
     parentNodeId,
     createNodeId,
     auth,
+    httpOptions,
     httpHeaders,
     ext,
     name,