fix: fix Contentful sync limit (#21644)

* fix: fix contentful sync limit * fix: use pluginOptions for default * fix: use pageLimit param instead * Update packages/gatsby-source-contentful/README.md Co-Authored-By: LB <laurie@gatsbyjs.com> Co-authored-by: LB <laurie@gatsbyjs.com>
gatsbyjs · Feb 21, 2020 · af921bb · af921bb
1 parent ad92e13
commit af921bb
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 8 deletions.
diff --git a/packages/gatsby-source-contentful/README.md b/packages/gatsby-source-contentful/README.md
@@ -168,9 +168,9 @@ Using the ID is a much more stable property to work with as it will change less
 
 If you are confident your Content Types will have natural-language IDs (e.g. `blogPost`), then you should set this option to `false`. If you are unable to ensure this, then you should leave this option set to `true` (the default).
 
-**`pageLimit`** [number][optional] [default: `1000`]
+**`pageLimit`** [number][optional] [default: `100`]
 
-Number of entries to retrieve from Contentful at a time. This can be adjusted to fix issues related to "Response size too big" error.
+Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g `50`.
 
 ## Notes on Contentful Content Models
 

diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json
@@ -13,7 +13,7 @@
     "base64-img": "^1.0.4",
     "bluebird": "^3.7.2",
     "chalk": "^2.4.2",
-    "contentful": "^6.1.3",
+    "contentful": "^7.14.0",
     "deep-map": "^1.5.0",
     "fs-extra": "^8.1.0",
     "gatsby-core-utils": "^1.0.28",

diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch.js b/packages/gatsby-source-contentful/src/__tests__/fetch.js
@@ -144,7 +144,7 @@ it(`calls contentful.getContentTypes with default page limit`, async () => {
 
   expect(reporter.panic).not.toBeCalled()
   expect(mockClient.getContentTypes).toHaveBeenCalledWith({
-    limit: 1000,
+    limit: 100,
     order: `sys.createdAt`,
     skip: 0,
   })

diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js
@@ -10,6 +10,7 @@ module.exports = async ({ syncToken, reporter, pluginConfig }) => {
 
   console.log(`Starting to fetch data from Contentful`)
 
+  const pageLimit = pluginConfig.get(`pageLimit`)
   const contentfulClientOptions = {
     space: pluginConfig.get(`spaceId`),
     accessToken: pluginConfig.get(`accessToken`),
@@ -75,7 +76,9 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`)
 
   let currentSyncData
   try {
-    let query = syncToken ? { nextSyncToken: syncToken } : { initial: true }
+    let query = syncToken
+      ? { nextSyncToken: syncToken }
+      : { initial: true, limit: pageLimit }
     currentSyncData = await client.sync(query)
   } catch (e) {
     reporter.panic(`Fetching contentful data failed`, e)
@@ -85,8 +88,6 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`)
   // doesn't support this.
   let contentTypes
   try {
-    const pageLimit = pluginConfig.get(`pageLimit`)
-
     contentTypes = await pagedGet(client, `getContentTypes`, pageLimit)
   } catch (e) {
     console.log(`error fetching content types`, e)

diff --git a/packages/gatsby-source-contentful/src/plugin-options.js b/packages/gatsby-source-contentful/src/plugin-options.js
@@ -3,7 +3,7 @@ const chalk = require(`chalk`)
 
 const _ = require(`lodash`)
 
-const DEFAULT_PAGE_LIMIT = 1000
+const DEFAULT_PAGE_LIMIT = 100
 
 const defaultOptions = {
   host: `cdn.contentful.com`,