[1.0] For Contentful: filter out unresolvable entries and create markdown text nodes #1202
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,8 @@ | ||
| const contentful = require(`contentful`) | ||
| const crypto = require(`crypto`) | ||
| const stringify = require("json-stringify-safe") | ||
|
|
||
| const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`) | ||
|
|
||
| const typePrefix = `contentful__` | ||
| const conflictFieldPrefix = `contentful` | ||
|
|
@@ -58,6 +61,19 @@ exports.sourceNodes = async ( | |
| console.log(`assets fetched`, assets.items.length) | ||
| console.timeEnd(`fetch Contentful data`) | ||
|
|
||
| // Create map of not resolvable ids so we can filter them out while creating | ||
| // links. | ||
| const notResolvable = new Map() | ||
| entryList.forEach(ents => { | ||
| if (ents.errors) { | ||
| ents.errors.forEach(error => { | ||
| if (error.sys.id === `notResolvable`) { | ||
| notResolvable.set(error.details.id, error.details) | ||
| } | ||
| }) | ||
| } | ||
| }) | ||
|
|
||
| const contentTypeItems = contentTypes.items | ||
|
|
||
| // Build foreign reference map before starting to insert any nodes | ||
|
|
@@ -75,6 +91,11 @@ exports.sourceNodes = async ( | |
| entryItemFieldValue[0].sys.id | ||
| ) { | ||
| entryItemFieldValue.forEach(v => { | ||
| // Don't create link to an unresolvable field. | ||
| if (notResolvable.has(v.sys.id)) { | ||
| return | ||
| } | ||
|
|
||
| if (!foreignReferenceMap[v.sys.id]) { | ||
| foreignReferenceMap[v.sys.id] = [] | ||
| } | ||
|
|
@@ -87,7 +108,8 @@ exports.sourceNodes = async ( | |
| } else if ( | ||
| entryItemFieldValue.sys && | ||
| entryItemFieldValue.sys.type && | ||
| entryItemFieldValue.sys.id && | ||
| !notResolvable.has(entryItemFieldValue.sys.id) | ||
| ) { | ||
| if (!foreignReferenceMap[entryItemFieldValue.sys.id]) { | ||
| foreignReferenceMap[entryItemFieldValue.sys.id] = [] | ||
|
|
@@ -101,6 +123,26 @@ exports.sourceNodes = async ( | |
| }) | ||
| }) | ||
|
|
||
| function createTextNode(node, text, createNode) { | ||
| const textNode = { | ||
| id: `${node.id}TextNode`, | ||
| parent: node.id, | ||
| children: [], | ||
| text, | ||
| internal: { | ||
| type: `ComponentDescription`, | ||
| mediaType: `text/x-markdown`, | ||
| content: text, | ||
| contentDigest: digest(text), | ||
| }, | ||
| } | ||
|
|
||
| node.children = node.children.concat([textNode.id]) | ||
| createNode(textNode) | ||
|
|
||
| return textNode.id | ||
| } | ||
|
|
||
| contentTypeItems.forEach((contentTypeItem, i) => { | ||
| const contentTypeItemId = contentTypeItem.sys.id | ||
|
|
||
|
|
@@ -138,13 +180,17 @@ exports.sourceNodes = async ( | |
| ) { | ||
| entryItemFields[ | ||
| `${entryItemFieldKey}___NODE` | ||
| ] = entryItemFieldValue | ||
| .filter(v => !notResolvable.has(v.sys.id)) | ||
| .map(v => v.sys.id) | ||
|
|
||
| delete entryItemFields[entryItemFieldKey] | ||
| } | ||
| } else if ( | ||
| entryItemFieldValue.sys && | ||
| entryItemFieldValue.sys.type && | ||
| entryItemFieldValue.sys.id && | ||
| !notResolvable.has(entryItemFieldValue.sys.id) | ||
| ) { | ||
| entryItemFields[`${entryItemFieldKey}___NODE`] = | ||
| entryItemFieldValue.sys.id | ||
|
|
@@ -167,30 +213,42 @@ exports.sourceNodes = async ( | |
| }) | ||
| } | ||
|
|
||
| let entryNode = { | ||
| id: entryItem.sys.id, | ||
| parent: contentTypeItemId, | ||
| children: [], | ||
| internal: { | ||
| type: `${makeTypeName(contentTypeItemId)}`, | ||
| mediaType: `application/json`, | ||
| }, | ||
| } | ||
|
|
||
| // Replace text fields with text nodes so we can process their markdown | ||
|
Contributor
There was a problem hiding this comment. Nice, in a separate project I ended up doing...
... with a text field using
Contributor
Author
There was a problem hiding this comment. Yeah, doing markdown conversion at compile time is much nicer. Much less client code + somewhat faster render times. |
||
| // into HTML. | ||
| Object.keys(entryItemFields).forEach(entryItemFieldKey => { | ||
| if (entryItemFieldKey === `text`) { | ||
| entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode( | ||
| entryNode, | ||
| entryItemFields[entryItemFieldKey], | ||
| createNode | ||
| ) | ||
|
|
||
| delete entryItemFields[entryItemFieldKey] | ||
| } | ||
| }) | ||
|
|
||
| entryNode = { ...entryItemFields, ...entryNode } | ||
|
|
||
| // Get content digest of node. | ||
| const contentDigest = digest(stringify(entryNode)) | ||
|
|
||
| entryNode.internal.contentDigest = contentDigest | ||
|
|
||
| return entryNode | ||
| }) | ||
|
|
||
| // Create a node for each content type | ||
| const contentTypeItemStr = stringify(contentTypeItem) | ||
|
|
||
| const contentTypeNode = { | ||
|
Contributor
There was a problem hiding this comment. Not really in scope for this review, but I wonder if it was a good idea to expose ContentType itself as a gatsby node. I haven't thought of a use case for it yet. I suppose there's no particular harm, but could consider removing it to lower the plugin complexity / surface area.
Contributor
Author
There was a problem hiding this comment. Yeah, I wondered about this too... One use case perhaps is someone building an internal documentation site from their content model. I'd lean towards keeping it I guess. I assume Contentful exposes it because there's sufficient demand? @Khaledgarbaya could you provide background for what people generally use this data for?
Contributor
There was a problem hiding this comment. Hi @KyleAMathews I've seen user generally get ContentTypes if they want to map content types to components, for example, to have some sort of dynamic fields etc. Bu the normal usage I see user mostly care about Entries and assets |
||
| id: contentTypeItemId, | ||
|
|
@@ -201,16 +259,12 @@ exports.sourceNodes = async ( | |
| description: contentTypeItem.description, | ||
| internal: { | ||
| type: `${makeTypeName(`ContentType`)}`, | ||
| mediaType: `text/x-contentful`, | ||
| }, | ||
| } | ||
|
|
||
| // Get content digest of node. | ||
| const contentDigest = digest(stringify(contentTypeNode)) | ||
|
|
||
| contentTypeNode.internal.contentDigest = contentDigest | ||
|
|
||
|
|
@@ -222,7 +276,7 @@ exports.sourceNodes = async ( | |
|
|
||
| assets.items.forEach(assetItem => { | ||
| // Create a node for each asset. They may be referenced by Entries | ||
| const assetItemStr = stringify(assetItem) | ||
|
|
||
| const assetNode = { | ||
| id: assetItem.sys.id, | ||
|
|
@@ -231,16 +285,12 @@ exports.sourceNodes = async ( | |
| ...assetItem.fields, | ||
| internal: { | ||
| type: `${makeTypeName(`Asset`)}`, | ||
| mediaType: `text/x-contentful`, | ||
| }, | ||
| } | ||
|
|
||
| // Get content digest of node. | ||
| const contentDigest = digest(stringify(assetNode)) | ||
|
|
||
| assetNode.internal.contentDigest = contentDigest | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Blanking right now.. why would there be non-resolvable ids?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The only place I found it documented is here: http://cocoadocs.org/docsets/ContentfulDeliveryAPI/1.9.2/Classes/CDAArray.html
Basically says if a referenced entry is deleted but the reference also isn't also deleted they send this error 🤷♂️
The client I'm working with has a ton of content types and has been doing a lot of experimenting so this came up.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Got it, cool. Good to use it with real existing sites. :)