For Contentful, filter out unresolvable entries and create markdown text nodes (#1202)

Author: KyleAMathews

docs/docs/plugins.md

@@ -94,6 +94,7 @@ you can place the files in a `src` subfolder and build them to the plugin folder
 * [gatsby-remark-responsive-image](/docs/packages/gatsby-remark-responsive-image/)
 * [gatsby-remark-smartypants](/docs/packages/gatsby-remark-smartypants/)
 * [gatsby-sharp](/docs/packages/gatsby-sharp/)
+* [gatsby-source-contentful](/docs/packages/gatsby-source-contentful/)
 * [gatsby-source-drupal](/docs/packages/gatsby-source-drupal/)
 * [gatsby-source-filesystem](/docs/packages/gatsby-source-filesystem/)
 * [gatsby-source-hacker-news](/docs/packages/gatsby-source-hacker-news/)

packages/gatsby-plugin-offline/src/app-shell.js

@@ -2,6 +2,8 @@ import React from "react"
 
 class AppShell extends React.Component {
   componentDidMount() {
+    // TODO check if page exists and if not,
+    // force a hard reload.
     window.___navigateTo(this.props.location.pathname)
   }
 

packages/gatsby-source-contentful/package.json

@@ -17,6 +17,7 @@
     "axios": "^0.16.1",
     "bluebird": "^3.4.6",
     "contentful": "^4.3.0",
+    "json-stringify-safe": "^5.0.1",
     "lodash": "^4.17.2"
   }
 }

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -1,5 +1,8 @@
 const contentful = require(`contentful`)
 const crypto = require(`crypto`)
+const stringify = require("json-stringify-safe")
+
+const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`)
 
 const typePrefix = `contentful__`
 const conflictFieldPrefix = `contentful`
@@ -58,6 +61,19 @@ exports.sourceNodes = async (
   console.log(`assets fetched`, assets.items.length)
   console.timeEnd(`fetch Contentful data`)
 
+  // Create map of not resolvable ids so we can filter them out while creating
+  // links.
+  const notResolvable = new Map()
+  entryList.forEach(ents => {
+    if (ents.errors) {
+      ents.errors.forEach(error => {
+        if (error.sys.id === `notResolvable`) {
+          notResolvable.set(error.details.id, error.details)
+        }
+      })
+    }
+  })
+
   const contentTypeItems = contentTypes.items
 
   // Build foreign reference map before starting to insert any nodes
@@ -75,6 +91,11 @@ exports.sourceNodes = async (
             entryItemFieldValue[0].sys.id
           ) {
             entryItemFieldValue.forEach(v => {
+              // Don't create link to an unresolvable field.
+              if (notResolvable.has(v.sys.id)) {
+                return
+              }
+
               if (!foreignReferenceMap[v.sys.id]) {
                 foreignReferenceMap[v.sys.id] = []
               }
@@ -87,7 +108,8 @@ exports.sourceNodes = async (
         } else if (
           entryItemFieldValue.sys &&
           entryItemFieldValue.sys.type &&
-          entryItemFieldValue.sys.id
+          entryItemFieldValue.sys.id &&
+          !notResolvable.has(entryItemFieldValue.sys.id)
         ) {
           if (!foreignReferenceMap[entryItemFieldValue.sys.id]) {
             foreignReferenceMap[entryItemFieldValue.sys.id] = []
@@ -101,6 +123,26 @@ exports.sourceNodes = async (
     })
   })
 
+  function createTextNode(node, text, createNode) {
+    const textNode = {
+      id: `${node.id}TextNode`,
+      parent: node.id,
+      children: [],
+      text,
+      internal: {
+        type: `ComponentDescription`,
+        mediaType: `text/x-markdown`,
+        content: text,
+        contentDigest: digest(text),
+      },
+    }
+
+    node.children = node.children.concat([textNode.id])
+    createNode(textNode)
+
+    return textNode.id
+  }
+
   contentTypeItems.forEach((contentTypeItem, i) => {
     const contentTypeItemId = contentTypeItem.sys.id
 
@@ -138,13 +180,17 @@ exports.sourceNodes = async (
           ) {
             entryItemFields[
               `${entryItemFieldKey}___NODE`
-            ] = entryItemFieldValue.map(v => v.sys.id)
+            ] = entryItemFieldValue
+              .filter(v => !notResolvable.has(v.sys.id))
+              .map(v => v.sys.id)
+
             delete entryItemFields[entryItemFieldKey]
           }
         } else if (
           entryItemFieldValue.sys &&
           entryItemFieldValue.sys.type &&
-          entryItemFieldValue.sys.id
+          entryItemFieldValue.sys.id &&
+          !notResolvable.has(entryItemFieldValue.sys.id)
         ) {
           entryItemFields[`${entryItemFieldKey}___NODE`] =
             entryItemFieldValue.sys.id
@@ -167,30 +213,42 @@ exports.sourceNodes = async (
         })
       }
 
-      const entryNode = {
+      let entryNode = {
         id: entryItem.sys.id,
         parent: contentTypeItemId,
         children: [],
-        ...entryItemFields,
         internal: {
           type: `${makeTypeName(contentTypeItemId)}`,
-          content: JSON.stringify(entryItem),
           mediaType: `application/json`,
         },
       }
 
+      // Replace text fields with text nodes so we can process their markdown
+      // into HTML.
+      Object.keys(entryItemFields).forEach(entryItemFieldKey => {
+        if (entryItemFieldKey === `text`) {
+          entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode(
+            entryNode,
+            entryItemFields[entryItemFieldKey],
+            createNode
+          )
+
+          delete entryItemFields[entryItemFieldKey]
+        }
+      })
+
+      entryNode = { ...entryItemFields, ...entryNode }
+
       // Get content digest of node.
-      const contentDigest = crypto
-        .createHash(`md5`)
-        .update(JSON.stringify(entryNode))
-        .digest(`hex`)
+      const contentDigest = digest(stringify(entryNode))
 
       entryNode.internal.contentDigest = contentDigest
 
       return entryNode
     })
+
     // Create a node for each content type
-    const contentTypeItemStr = JSON.stringify(contentTypeItem)
+    const contentTypeItemStr = stringify(contentTypeItem)
 
     const contentTypeNode = {
       id: contentTypeItemId,
@@ -201,16 +259,12 @@ exports.sourceNodes = async (
       description: contentTypeItem.description,
       internal: {
         type: `${makeTypeName(`ContentType`)}`,
-        content: contentTypeItemStr,
-        mediaType: `application/json`,
+        mediaType: `text/x-contentful`,
       },
     }
 
     // Get content digest of node.
-    const contentDigest = crypto
-      .createHash(`md5`)
-      .update(JSON.stringify(contentTypeNode))
-      .digest(`hex`)
+    const contentDigest = digest(stringify(contentTypeNode))
 
     contentTypeNode.internal.contentDigest = contentDigest
 
@@ -222,7 +276,7 @@ exports.sourceNodes = async (
 
   assets.items.forEach(assetItem => {
     // Create a node for each asset. They may be referenced by Entries
-    const assetItemStr = JSON.stringify(assetItem)
+    const assetItemStr = stringify(assetItem)
 
     const assetNode = {
       id: assetItem.sys.id,
@@ -231,16 +285,12 @@ exports.sourceNodes = async (
       ...assetItem.fields,
       internal: {
         type: `${makeTypeName(`Asset`)}`,
-        content: assetItemStr,
-        mediaType: `application/json`,
+        mediaType: `text/x-contentful`,
       },
     }
 
     // Get content digest of node.
-    const contentDigest = crypto
-      .createHash(`md5`)
-      .update(JSON.stringify(assetNode))
-      .digest(`hex`)
+    const contentDigest = digest(stringify(assetNode))
 
     assetNode.internal.contentDigest = contentDigest
 

packages/gatsby/package.json

@@ -55,6 +55,7 @@
     "is-relative-url": "^2.0.0",
     "joi": "^9.1.1",
     "json-loader": "^0.5.2",
+    "json-stringify-safe": "^5.0.1",
     "json5": "^0.5.0",
     "loader-utils": "^0.2.16",
     "lodash": "^4.17.4",

packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js

@@ -35,7 +35,7 @@ class Dev404Page extends React.Component {
             <h2>Pages ({this.props.data.allSitePage.totalCount})</h2>
             <ul>
               {this.props.data.allSitePage.edges.map(({ node }) =>
-                <li><Link to={node.path}>{node.path}</Link></li>
+                <li key={node.path}><Link to={node.path}>{node.path}</Link></li>
               )}
             </ul>
           </div>}

packages/gatsby/src/redux/index.js

@@ -4,6 +4,7 @@ const _ = require(`lodash`)
 const { composeWithDevTools } = require(`remote-redux-devtools`)
 const fs = require(`fs`)
 const EventEmitter = require(`eventemitter2`)
+const stringify = require(`json-stringify-safe`)
 
 // Create event emitter for actions
 const emitter = new EventEmitter()
@@ -47,7 +48,7 @@ const saveState = _.debounce(state => {
   const pickedState = _.pick(state, [`nodes`, `status`, `pageDataDependencies`])
   fs.writeFile(
     `${process.cwd()}/.cache/redux-state.json`,
-    JSON.stringify(pickedState, null, 2),
+    stringify(pickedState, null, 2),
     () => {}
   )
 }, 1000)

packages/gatsby/src/schema/infer-graphql-type.js

@@ -253,6 +253,7 @@ function inferFromFieldName(value, selector, types): GraphQLFieldConfig<*, *> {
   const [, , linkedField] = key.split(`___`)
 
   const linkedNode = findLinkedNode(value, linkedField)
+
   invariant(
     linkedNode,
     oneLine`
@@ -464,9 +465,10 @@ export function inferObjectStructureFromNodes({
       shouldInferFile(nodes, nextSelector, value)
     ) {
       inferredField = inferFromUri(key, types)
+    }
 
-      // Finally our automatic inference of field value type.
-    } else {
+    // Finally our automatic inference of field value type.
+    if (!inferredField) {
       inferredField = inferGraphQLType({
         nodes,
         types,