Skip to content

Commit 56b467b

Browse files
wardpeetwardpeet
committed
add integration test
1 parent 12d2fdc commit 56b467b

File tree

8 files changed

+272
-10
lines changed

8 files changed

+272
-10
lines changed

‎integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/**
2+
* We want to make sure that fetch-remote-file is working with multi workers.
3+
*/
4+
5+
const execa = require(`execa`)
6+
const path = require(`path`)
7+
const glob = require(`glob`)
8+
const fs = require(`fs-extra`)
9+
const md5File = require(`md5-file`)
10+
const basePath = path.resolve(__dirname, `../../`)
11+
12+
const cleanDirs = () =>
13+
Promise.all([
14+
fs.emptyDir(`${basePath}/public`),
15+
fs.emptyDir(`${basePath}/.cache`),
16+
])
17+
18+
describe(`fetch-remote-file`, () => {
19+
beforeAll(async () => {
20+
await cleanDirs()
21+
await execa(`yarn`, [`build`], {
22+
cwd: basePath,
23+
// we want to force 1 query per worker
24+
env: { NODE_ENV: `production`, GATSBY_PARALLEL_QUERY_CHUNK_SIZE: `1` },
25+
})
26+
}, 60 * 1000)
27+
28+
it("should have the correct md5", async () => {
29+
expect(
30+
await md5File(
31+
path.join(
32+
__dirname,
33+
"../..",
34+
"public/images/50c58a791de3c2303e62084d731799eb/photoA.jpg"
35+
)
36+
)
37+
).toEqual("a9e57a66a10b2d26a1999a4685d7c9ef")
38+
expect(
39+
await md5File(
40+
path.join(
41+
__dirname,
42+
"../..",
43+
"public/images/4910e745c3c453b8795d6ba65c79d99b/photoB.jpg"
44+
)
45+
)
46+
).toEqual("c305dc5c5db45cc773231a507af5116d")
47+
expect(
48+
await md5File(
49+
path.join(
50+
__dirname,
51+
"../..",
52+
"public/images/fb673e75e9534b3cc2d2e24085386d48/photoC.jpg"
53+
)
54+
)
55+
).toEqual("4ba953ba27236727d7abe7d5b8916432")
56+
})
57+
58+
it("should have conflict between workers", async () => {
59+
const files = await fs.readdir(path.join(__dirname, "../../.cache/workers"))
60+
61+
expect(files.length).toBe(4)
62+
})
63+
})

‎integration-tests/gatsby-pipeline/gatsby-node.js‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
const { fetchRemoteFile } = require("gatsby-core-utils/fetch-remote-file")
2+
const { slash } = require("gatsby-core-utils")
3+
const path = require("path")
4+
const fs = require("fs-extra")
5+
6+
/** @type{import('gatsby').createSchemaCustomization} */
7+
exports.createSchemaCustomization = ({ actions, schema, cache, reporter }) => {
8+
actions.createTypes(
9+
schema.buildObjectType({
10+
name: "MyRemoteFile",
11+
fields: {
12+
url: "String!",
13+
publicUrl: {
14+
type: "String!",
15+
async resolve(source) {
16+
const filePath = await fetchRemoteFile({
17+
name: path.basename(source.name, path.extname(source.name)),
18+
ext: path.extname(source.name),
19+
url: source.url,
20+
directory: "./public/images",
21+
})
22+
23+
const dir = path.join(global.__GATSBY.root, ".cache", "workers")
24+
await fs.ensureDir(dir)
25+
await fs.createFile(
26+
`${path.join(dir, `worker-${process.env.GATSBY_WORKER_ID}`)}`
27+
)
28+
29+
const workers = (await cache.get("workers")) ?? []
30+
workers.push(process.env.GATSBY_WORKER_ID)
31+
32+
return `${slash(filePath.replace(/^public/, ""))}`
33+
},
34+
},
35+
},
36+
interfaces: ["Node"],
37+
})
38+
)
39+
}
40+
41+
/** @type {imporg('gatsby').sourceNodes} */
42+
exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => {
43+
const items = [
44+
{
45+
name: "photoA.jpg",
46+
url: "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
47+
},
48+
{
49+
name: "photoB.jpg",
50+
url: "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
51+
},
52+
{
53+
name: "photoC.jpg",
54+
url: "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
55+
},
56+
]
57+
58+
items.forEach((item, index) => {
59+
actions.createNode({
60+
id: createNodeId(`remote-file-${index}`),
61+
name: item.name,
62+
url: item.url,
63+
internal: {
64+
type: "MyRemoteFile",
65+
contentDigest: createContentDigest(item.url),
66+
},
67+
})
68+
})
69+
}

‎integration-tests/gatsby-pipeline/package.json‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
"version": "1.0.0",
55
"author": "Kyle Mathews <[email protected]>",
66
"dependencies": {
7-
"gatsby": "latest",
7+
"gatsby": "4.8.0-next.1-dev-1645011114818",
88
"gatsby-image": "latest",
9-
"gatsby-plugin-gatsby-cloud": "latest",
10-
"gatsby-plugin-react-helmet": "latest",
11-
"gatsby-plugin-sharp": "latest",
12-
"gatsby-source-filesystem": "latest",
13-
"gatsby-transformer-sharp": "latest",
9+
"gatsby-plugin-gatsby-cloud": "4.8.0-next.1-dev-1645011114818",
10+
"gatsby-plugin-react-helmet": "5.8.0-next.0-dev-1645011114818",
11+
"gatsby-plugin-sharp": "4.8.0-next.0-dev-1645011114818",
12+
"gatsby-source-filesystem": "4.8.0-next.0-dev-1645011114818",
13+
"gatsby-transformer-sharp": "4.8.0-next.0-dev-1645011114818",
1414
"prop-types": "^15.7.2",
1515
"react": "^17.0.2",
1616
"react-dom": "^17.0.2",
@@ -26,11 +26,12 @@
2626
"test": "jest"
2727
},
2828
"devDependencies": {
29+
"cross-env": "^7.0.3",
2930
"execa": "^4.0.3",
3031
"fs-extra": "^9.0.1",
32+
"jest": "^27.2.1",
3133
"md5-file": "^5.0.0",
3234
"node-fetch": "^2.6.0",
33-
"jest": "^27.2.1",
3435
"tree-kill": "^1.2.2"
3536
},
3637
"repository": {
@@ -40,4 +41,4 @@
4041
"engines": {
4142
"node": ">=12.13.0"
4243
}
43-
}
44+
}

‎integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import React from "react"
2+
import { graphql, Link } from "gatsby"
3+
4+
import Layout from "../components/layout"
5+
import SEO from "../components/seo"
6+
7+
const FetchRemoteA = ({ data }) => {
8+
console.log({ data })
9+
return (
10+
<Layout>
11+
<SEO title="Fetch Remote A" />
12+
13+
<pre
14+
dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
15+
/>
16+
<Link to="/">Go back to the homepage</Link>
17+
</Layout>
18+
)
19+
}
20+
21+
export default FetchRemoteA
22+
23+
export const pageQuery = graphql`
24+
{
25+
allMyRemoteFile {
26+
nodes {
27+
url
28+
publicUrl
29+
}
30+
}
31+
}
32+
`

‎integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import React from "react"
2+
import { graphql, Link } from "gatsby"
3+
4+
import Layout from "../components/layout"
5+
import SEO from "../components/seo"
6+
7+
const FetchRemoteB = ({ data }) => {
8+
return (
9+
<Layout>
10+
<SEO title="Fetch Remote B" />
11+
12+
<pre
13+
dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
14+
/>
15+
16+
<Link to="/">Go back to the homepage</Link>
17+
</Layout>
18+
)
19+
}
20+
21+
export default FetchRemoteB
22+
23+
export const pageQuery = graphql`
24+
{
25+
allMyRemoteFile {
26+
nodes {
27+
url
28+
publicUrl
29+
}
30+
}
31+
}
32+
`

‎integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import React from "react"
2+
import { graphql, Link } from "gatsby"
3+
4+
import Layout from "../components/layout"
5+
import SEO from "../components/seo"
6+
7+
const FetchRemoteB = ({ data }) => {
8+
return (
9+
<Layout>
10+
<SEO title="Fetch Remote C" />
11+
12+
<pre
13+
dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
14+
/>
15+
<Link to="/">Go back to the homepage</Link>
16+
</Layout>
17+
)
18+
}
19+
20+
export default FetchRemoteB
21+
22+
export const pageQuery = graphql`
23+
{
24+
allMyRemoteFile {
25+
nodes {
26+
url
27+
publicUrl
28+
}
29+
}
30+
}
31+
`

‎integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import React from "react"
2+
import { graphql, Link } from "gatsby"
3+
4+
import Layout from "../components/layout"
5+
import SEO from "../components/seo"
6+
7+
const FetchRemoteB = ({ data }) => {
8+
return (
9+
<Layout>
10+
<SEO title="Fetch Remote B" />
11+
12+
<pre
13+
dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
14+
/>
15+
<Link to="/">Go back to the homepage</Link>
16+
</Layout>
17+
)
18+
}
19+
20+
export default FetchRemoteB
21+
22+
export const pageQuery = graphql`
23+
{
24+
allMyRemoteFile {
25+
nodes {
26+
url
27+
publicUrl
28+
}
29+
}
30+
}
31+
`

‎packages/gatsby-core-utils/src/fetch-remote-file.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
getRemoteFileExtension,
99
createFilePath,
1010
} from "./filename-utils"
11+
import { slash } from "./path"
1112
import { requestRemoteNode } from "./remote-file-utils/fetch-file"
1213
import { getStorage, getDatabaseDir } from "./utils/get-storage"
1314
import { createMutex } from "./mutex"
@@ -182,7 +183,9 @@ async function fetchFile({
182183
httpOptions
183184
)
184185

185-
const filename = createFilePath(path.join(fileDirectory, digest), name, ext)
186+
const filename = slash(
187+
createFilePath(path.join(fileDirectory, digest), name, ext)
188+
)
186189
if (response.statusCode === 200) {
187190
// Save the response headers for future requests.
188191
// If the user did not provide an extension and we couldn't get one from remote file, try and guess one
@@ -200,7 +203,7 @@ async function fetchFile({
200203
cacheKey,
201204
extension: ext,
202205
headers: response.headers.etag ? { etag: response.headers.etag } : {},
203-
directory: fileDirectory,
206+
directory: slash(fileDirectory),
204207
path: filename.replace(fileDirectory, ``),
205208
})
206209
} else if (response.statusCode === 304) {

0 commit comments

Comments
 (0)