[WIP] : Use the sync endpoint to pull data from Contentful (gatsbyjs#…

…1241)

* Initial commit

* Initial commit

* Fixes

* get default locale

* Update image schema

* Fix broken code

* fixup! Fix broken code

* fixup! fixup! Fix broken code

* Update Snapshot

packages/gatsby-source-contentful/proxy-handlers.js

@@ -0,0 +1,12 @@
+"use strict";
+
+var localeProxyHandler = {
+  get: function get(target, prop) {
+    if (prop in target[prop] && target.defaultLocale in target[prop]) {
+      return target[prop][target.defaultLocale];
+    }
+    return target[prop];
+  }
+};
+
+exports.localeProxyHandler = localeProxyHandler;

packages/gatsby-source-contentful/src/__tests__/__snapshots__/extend-node-type.js.snap

@@ -6,7 +6,7 @@ exports[`contentful extend node type createUrl ignores options it doesn't unders
 
 exports[`contentful extend node type resolveResize generates resized images 1`] = `
 Object {
-  "aspectRatio": 0.7499625018749062,
+  "aspectRatio": 0.75,
   "base64": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAlgCWAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAbABQDAREAAhEBAxEB/8QAGQABAAIDAAAAAAAAAAAAAAAABgIHAwQJ/8QALhAAAQMDAgIHCQAAAAAAAAAAAQIDBAAFEQYSEzEHFCEiQVFhFRYkMkJDcXKj/8QAGQEAAgMBAAAAAAAAAAAAAAAAAQIAAwQF/8QAJREAAQQCAQEJAAAAAAAAAAAAAQACAxEEQSEiEhMxQlFhcYGh/9oADAMBAAIRAxEAPwDp4Zz3vV1Tf8OIXF2YHzb8Zzz5UNpL6qWixdparPY3lO5clSUIdVtHeSQo49OQoXwl7RoFJaZWo8tWNcqHh7Mz/U0u0nnRF2+O3DRmiJtilRJLDtyY4jzm4oLQDgXtx9WRjyrSGNhcWZAINfuvpcl078mCOXBc0guFk3VAkOqtqz6oXZQK7amhQ+lJy0rcUJy9PrlIRtOCgOqBOeXMVZ3TywzaHCwuyo2ZTccnqLSR8A+qPR2o9v0nYLbGQhhqM68oIbSAEYX4Afuazue6TlxspYo2RRNjjAAF+CtiI+JUVl4fcQFdnqKsW8cqKrfGXN62plJk8Is8Qjt2E52/jNH2QoXamIrIxhlsY5d0dlBMsgAAwBgeQqKL/9k=",
   "height": 533,
   "src": "//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg?w=400",
@@ -26,7 +26,7 @@ Object {
 
 exports[`contentful extend node type resolveResponsiveResolution generates responsive resolution data for images 1`] = `
 Object {
-  "aspectRatio": 0.7499625018749062,
+  "aspectRatio": 0.75,
   "base64": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAlgCWAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAbABQDAREAAhEBAxEB/8QAGQABAAIDAAAAAAAAAAAAAAAABgIHAwQJ/8QALhAAAQMDAgIHCQAAAAAAAAAAAQIDBAAFEQYSEzEHFCEiQVFhFRYkMkJDcXKj/8QAGQEAAgMBAAAAAAAAAAAAAAAAAQIAAwQF/8QAJREAAQQCAQEJAAAAAAAAAAAAAQACAxEEQSEiEhMxQlFhcYGh/9oADAMBAAIRAxEAPwDp4Zz3vV1Tf8OIXF2YHzb8Zzz5UNpL6qWixdparPY3lO5clSUIdVtHeSQo49OQoXwl7RoFJaZWo8tWNcqHh7Mz/U0u0nnRF2+O3DRmiJtilRJLDtyY4jzm4oLQDgXtx9WRjyrSGNhcWZAINfuvpcl078mCOXBc0guFk3VAkOqtqz6oXZQK7amhQ+lJy0rcUJy9PrlIRtOCgOqBOeXMVZ3TywzaHCwuyo2ZTccnqLSR8A+qPR2o9v0nYLbGQhhqM68oIbSAEYX4Afuazue6TlxspYo2RRNjjAAF+CtiI+JUVl4fcQFdnqKsW8cqKrfGXN62plJk8Is8Qjt2E52/jNH2QoXamIrIxhlsY5d0dlBMsgAAwBgeQqKL/9k=",
   "height": 533,
   "src": "//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg?w=400",
@@ -54,7 +54,7 @@ Object {
 
 exports[`contentful extend node type resolveResponsiveSizes generates responsive size data for images 1`] = `
 Object {
-  "aspectRatio": 0.7499625018749062,
+  "aspectRatio": 0.75,
   "base64": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAlgCWAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAbABQDAREAAhEBAxEB/8QAGQABAAIDAAAAAAAAAAAAAAAABgIHAwQJ/8QALhAAAQMDAgIHCQAAAAAAAAAAAQIDBAAFEQYSEzEHFCEiQVFhFRYkMkJDcXKj/8QAGQEAAgMBAAAAAAAAAAAAAAAAAQIAAwQF/8QAJREAAQQCAQEJAAAAAAAAAAAAAQACAxEEQSEiEhMxQlFhcYGh/9oADAMBAAIRAxEAPwDp4Zz3vV1Tf8OIXF2YHzb8Zzz5UNpL6qWixdparPY3lO5clSUIdVtHeSQo49OQoXwl7RoFJaZWo8tWNcqHh7Mz/U0u0nnRF2+O3DRmiJtilRJLDtyY4jzm4oLQDgXtx9WRjyrSGNhcWZAINfuvpcl078mCOXBc0guFk3VAkOqtqz6oXZQK7amhQ+lJy0rcUJy9PrlIRtOCgOqBOeXMVZ3TywzaHCwuyo2ZTccnqLSR8A+qPR2o9v0nYLbGQhhqM68oIbSAEYX4Afuazue6TlxspYo2RRNjjAAF+CtiI+JUVl4fcQFdnqKsW8cqKrfGXN62plJk8Is8Qjt2E52/jNH2QoXamIrIxhlsY5d0dlBMsgAAwBgeQqKL/9k=",
   "sizes": "(max-width: 400px) 100vw, 400px",
   "src": "//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg?w=400",
@@ -70,7 +70,7 @@ Object {
 
 exports[`contentful extend node type resolveResponsiveSizes generates responsive sizes data for images using all options 1`] = `
 Object {
-  "aspectRatio": 0.7499625018749062,
+  "aspectRatio": 0.75,
   "base64": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAlgCWAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAbABQDAREAAhEBAxEB/8QAGQABAAIDAAAAAAAAAAAAAAAABgIHAwQJ/8QALhAAAQMDAgIHCQAAAAAAAAAAAQIDBAAFEQYSEzEHFCEiQVFhFRYkMkJDcXKj/8QAGQEAAgMBAAAAAAAAAAAAAAAAAQIAAwQF/8QAJREAAQQCAQEJAAAAAAAAAAAAAQACAxEEQSEiEhMxQlFhcYGh/9oADAMBAAIRAxEAPwDp4Zz3vV1Tf8OIXF2YHzb8Zzz5UNpL6qWixdparPY3lO5clSUIdVtHeSQo49OQoXwl7RoFJaZWo8tWNcqHh7Mz/U0u0nnRF2+O3DRmiJtilRJLDtyY4jzm4oLQDgXtx9WRjyrSGNhcWZAINfuvpcl078mCOXBc0guFk3VAkOqtqz6oXZQK7amhQ+lJy0rcUJy9PrlIRtOCgOqBOeXMVZ3TywzaHCwuyo2ZTccnqLSR8A+qPR2o9v0nYLbGQhhqM68oIbSAEYX4Afuazue6TlxspYo2RRNjjAAF+CtiI+JUVl4fcQFdnqKsW8cqKrfGXN62plJk8Is8Qjt2E52/jNH2QoXamIrIxhlsY5d0dlBMsgAAwBgeQqKL/9k=",
   "sizes": "(max-width: 450px) 100vw, 450px",
   "src": "//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg?w=450&h=399&q=50",

packages/gatsby-source-contentful/src/__tests__/extend-node-type.js

@@ -20,18 +20,19 @@ describe(`contentful extend node type`, () => {
   })
 
   const image = {
+    defaultLocale: `en-US`,
     file: {
-      url: `//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg`,
-      fileName: `ryugj83mqwa1asojwtwb.jpg`,
-      contentType: `image/jpeg`,
-      details: {
-        size: 28435,
-        image: {
-          width: `4500`,
-          height: `6000.3`,
+        url: `//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg`,
+        fileName: `ryugj83mqwa1asojwtwb.jpg`,
+        contentType: `image/jpeg`,
+        details: {
+          size: 28435,
+          image: {
+            width: `4500`,
+            height: `6000`,
+          },
         },
       },
-    },
   }
   describe(`resolveResponsiveResolution`, () => {
     it(`generates responsive resolution data for images`, async () => {

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -1,84 +1,126 @@
 const contentful = require(`contentful`)
-const crypto = require(`crypto`)
-const _ = require(`lodash`)
-
-const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`)
 
 const processAPIData = require(`./process-api-data`)
 
 const conflictFieldPrefix = `contentful`
+
 // restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/
 const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`]
 
+const _ = require(`lodash`)
+
 exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType
 
 exports.sourceNodes = async (
   { boundActionCreators, getNode, hasNodeChanged, store },
   { spaceId, accessToken }
 ) => {
-  const { createNode } = boundActionCreators
-
+  const { createNode, setPluginStatus } = boundActionCreators
+  
   // Fetch articles.
-  console.time(`fetch Contentful data`)
+  console.time(`Fetch Contentful data`)
   console.log(`Starting to fetch data from Contentful`)
 
   const client = contentful.createClient({
     space: spaceId,
     accessToken,
   })
+  // the structure of the sync payload is a bit different
+  // all field will be in this format { fieldName: {'locale': value} } so we need to get the space and its default local
+  // this can be extended later to support multiple locals
+  let space
+  let defaultLocale = `en-US`
+  try {
+    console.log(`Fetching default locale`)
+    space = await client.getSpace()
+    defaultLocale = _.find(space.locales, { 'default': true }).code
+    console.log(`default local is : ${defaultLocale}`)
+  } catch (e) {
+    console.log(`can't get space`)
+    // TODO maybe return here
+  }
+
+  // Get sync token if it exists
+  let syncToken
+  let lastSyncedData = { entries: [], assets: [], deletedEntries: [], deletedAssets: [], nextSyncToken: null }
+  if (
+    store.getState().status.plugins &&
+    store.getState().status.plugins[`gatsby-source-contentful`]
+  ) {
+    lastSyncedData = store.getState().status.plugins[`gatsby-source-contentful`].status
+      .lastSyncedData
+    syncToken = lastSyncedData.nextSyncToken
+  }
+
+  // The SDK will map the entities to the following object
+  // {
+  //  entries,
+  //  assets,
+  //  deletedEntries,
+  //  deletedAssets
+  // }
+  let currentSyncData
+  try {
+    let query = syncToken ? { nextSyncToken: syncToken } : { initial: true }
+    currentSyncData = await client.sync(query) 
+  } catch (e) {
+    currentSyncData = { entries: [], assets: [], deletedEntries: [], deletedAssets:[] }
+    console.log(`error fetching contentful data`, e)
+  }
 
+  // We fetch content types normaly since we don't receive it in the sync data
   let contentTypes
   try {
-    contentTypes = await client.getContentTypes({ limit: 1000 })
+    contentTypes = await pagedGet(client, `getContentTypes`)
   } catch (e) {
     console.log(`error fetching content types`, e)
   }
   console.log(`contentTypes fetched`, contentTypes.items.length)
-  if (contentTypes.total > 1000) {
-    console.log(
-      `HI! gatsby-source-plugin isn't setup yet to paginate over 1000 content types (the max we can fetch in one go). Please help out the project and contribute a PR fixing this.`
+
+  const contentTypeItems = contentTypes.items
+  // remove outdated entries 
+  lastSyncedData.entries.filter(entry => {
+    return _.find(
+      currentSyncData.entries, (newEntry) => newEntry.sys.id === entry.sys.id
+      ||
+        _.find(
+          currentSyncData.deletedEntries, (deletedEntry) => deletedEntry.sys.id === entry.sys.id
+        )
     )
-  }
+  })
 
-  const entryList = await Promise.all(
-    contentTypes.items.map(async contentType => {
-      const contentTypeId = contentType.sys.id
-      let entries
-      try {
-        entries = await client.getEntries({
-          content_type: contentTypeId,
-          limit: 1000,
-        })
-      } catch (e) {
-        console.log(`error fetching entries`, e)
-      }
-      console.log(
-        `entries fetched for content type ${contentType.name} (${contentTypeId})`,
-        entries.items.length
+  // merge entries
+  lastSyncedData.entries = lastSyncedData.entries.concat(currentSyncData.entries)
+  let entryList = contentTypeItems.map(contentType => {
+    return lastSyncedData.entries.filter(entry => entry.sys.contentType.sys.id === contentType.sys.id )
+  })
+
+  lastSyncedData.assets.filter(asset => {
+    return _.find(
+        currentSyncData.assets, (newAsset) => newAsset.sys.id === asset.sys.id
       )
-      if (entries.total > 1000) {
-        console.log(
-          `HI! gatsby-source-plugin isn't setup yet to paginate over 1000 entries (the max we can fetch in one go). Please help out the project and contribute a PR fixing this.`
+      ||
+        _.find(
+          currentSyncData.deletedAssets, (deletedAsset) => deletedAsset.sys.id === asset.sys.id
         )
-      }
-
-      return entries
-    })
-  )
+  })
 
-  let assets
-  try {
-    assets = await client.getAssets({ limit: 1000 })
-  } catch (e) {
-    console.log(`error fetching assets`, e)
-  }
-  if (assets.total > 1000) {
-    console.log(
-      `HI! gatsby-source-plugin isn't setup yet to paginate over 1000 assets (the max we can fetch in one go). Please help out the project and contribute a PR fixing this.`
-    )
-  }
-  console.log(`assets fetched`, assets.items.length)
+  lastSyncedData.assets = lastSyncedData.assets.concat(currentSyncData.assets)
+  let assets = lastSyncedData.assets
+
+  console.log(`Total assets `, assets.length)
+  console.log(`Updated assets `, currentSyncData.assets.length)
+  console.log(`Deleted assets `, currentSyncData.deletedAssets.length)
   console.timeEnd(`fetch Contentful data`)
+
+  // update syncToken
+  lastSyncedData.nextSyncToken = currentSyncData.nextSyncToken
+  // cache the data
+  setPluginStatus({
+    status: {
+      lastSyncedData: lastSyncedData,
+    },
+  })
 
   // Create map of not resolvable ids so we can filter them out while creating
   // links.
@@ -93,15 +135,17 @@ exports.sourceNodes = async (
     }
   })
 
-  const contentTypeItems = contentTypes.items
-
-  // Build foreign reference map before starting to insert any nodes
+ // entryList = entryList.map(entries => entries.map(entryItem => {
+   // entryItem.defaultLocale = defaultLocale
+   // return new Proxy(entryItem, localeProxyHandler)
+ // }))
+// Build foreign reference map before starting to insert any nodes
   const foreignReferenceMap = processAPIData.buildForeignReferenceMap({
     contentTypeItems,
     entryList,
     notResolvable,
+    defaultLocale,
   })
-
   contentTypeItems.forEach((contentTypeItem, i) => {
     processAPIData.createContentTypeNodes({
       contentTypeItem,
@@ -111,12 +155,37 @@ exports.sourceNodes = async (
       createNode,
       notResolvable,
       foreignReferenceMap,
+      defaultLocale,
     })
   })
 
-  assets.items.forEach(assetItem => {
-    processAPIData.createAssetNodes({ assetItem, createNode })
+  assets.forEach(assetItem => {
+    processAPIData.createAssetNodes({ assetItem, createNode, defaultLocale })
   })
 
   return
 }
+/**
+ * Gets all the existing entities based on pagination parameters.
+ * The first call will have no aggregated response. Subsequent calls will
+ * concatenate the new responses to the original one.
+ */
+function pagedGet (client, method, query = {}, skip = 0, pageLimit = 1000, aggregatedResponse = null) {
+  return client[method]({
+    ...query,
+    skip: skip,
+    limit: pageLimit,
+    order: `sys.createdAt`,
+  })
+  .then((response) => {
+    if (!aggregatedResponse) {
+      aggregatedResponse = response
+    } else {
+      aggregatedResponse.items = aggregatedResponse.items.concat(response.items)
+    }
+    if (skip + pageLimit <= response.total) {
+      return pagedGet(client, method, skip + pageLimit, aggregatedResponse)
+    }
+    return aggregatedResponse
+  })
+}

packages/gatsby-source-contentful/src/process-api-data.js

@@ -10,14 +10,16 @@ exports.buildForeignReferenceMap = ({
   contentTypeItems,
   entryList,
   notResolvable,
+  defaultLocale,
 }) => {
   const foreignReferenceMap = {}
   contentTypeItems.forEach((contentTypeItem, i) => {
     const contentTypeItemId = contentTypeItem.sys.id
-    entryList[i].items.forEach(entryItem => {
+    entryList[i].forEach(entryItem => {
       const entryItemFields = entryItem.fields
       Object.keys(entryItemFields).forEach(entryItemFieldKey => {
-        const entryItemFieldValue = entryItemFields[entryItemFieldKey]
+        let entryItemFieldValue = entryItemFields[entryItemFieldKey][defaultLocale]
+        console.log(`${entryItemFieldKey} => ${String(entryItemFieldValue)}`)
         if (Array.isArray(entryItemFieldValue)) {
           if (
             entryItemFieldValue[0].sys &&
@@ -89,6 +91,7 @@ exports.createContentTypeNodes = ({
   createNode,
   notResolvable,
   foreignReferenceMap,
+  defaultLocale,
 }) => {
   const contentTypeItemId = contentTypeItem.sys.id
 
@@ -105,10 +108,10 @@ exports.createContentTypeNodes = ({
   })
 
   // First create nodes for each of the entries of that content type
-  const entryNodes = entries.items.map(entryItem => {
+  const entryNodes = entries.map(entryItem => {
     // Prefix any conflicting fields
     // https://github.com/gatsbyjs/gatsby/pull/1084#pullrequestreview-41662888
-    const entryItemFields = Object.assign({}, entryItem.fields)
+    const entryItemFields = entryItem.fields
     conflictFields.forEach(conflictField => {
       entryItemFields[`${conflictFieldPrefix}${conflictField}`] =
         entryItemFields[conflictField]
@@ -185,10 +188,11 @@ exports.createContentTypeNodes = ({
             : f.id) === entryItemFieldKey
       ).type
       if (fieldType === `Text`) {
+        console.log(entryItemFields[entryItemFieldKey])
         entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode(
           entryNode,
           entryItemFieldKey,
-          entryItemFields[entryItemFieldKey],
+          entryItemFields[entryItemFieldKey][defaultLocale],
           createNode
         )
 
@@ -231,8 +235,14 @@ exports.createContentTypeNodes = ({
   })
 }
 
-exports.createAssetNodes = ({ assetItem, createNode }) => {
+exports.createAssetNodes = ({ assetItem, createNode, defaultLocale }) => {
   // Create a node for each asset. They may be referenced by Entries
+  assetItem.fields = {
+    file: assetItem.fields.file[defaultLocale],
+    title: assetItem.fields.title[defaultLocale],
+    description: assetItem.fields.description[defaultLocale],
+  }
+  console.log(assetItem)
   const assetNode = {
     id: assetItem.sys.id,
     parent: `__SOURCE__`,

packages/gatsby-source-contentful/src/proxy-handlers.js

@@ -0,0 +1,13 @@
+const localeProxyHandler = {
+  get: (target, prop) => {
+    if (prop in target[prop] && target.defaultLocale in target[prop]) {
+      return target[prop][target.defaultLocale]
+    }
+    return target[prop]
+  },
+}
+
+
+exports.localeProxyHandler = localeProxyHandler
+
+