feat(gatsby-source-drupal): Image CDN support (#35265) (#35510)

Co-authored-by: Tyler Barnes <[email protected]>
Co-authored-by: Michal Piechowiak <[email protected]>

Author: 3 people

packages/gatsby-source-drupal/README.md

@@ -40,6 +40,68 @@ count in collection queries" `/admin/config/services/jsonapi/extras` as that
 [speeds up fetching data from Drupal by around
 4x](https://github.com/gatsbyjs/gatsby/pull/32883).
 
+### Gatsby Image CDN
+
+Gatsby has an Image CDN feature which speeds up your build times as well as your frontend performance.
+
+Previously Gatsby would fetch all image files during the Gatsby build process, transform them for frontend performance, and then serve them as static files on the frontend.
+With the new Image CDN feature images are lazily processed when users visit the frontend of your site. The first front-end visitor of any image will transform that image and cache it for all other users.
+Note that Image CDN works on all hosting platforms, but only speeds up your builds on Gatsby Cloud, as Gatsby Cloud is the most advanced CI/CD and hosting platform for the Gatsby framework.
+
+- [Image CDN blog post](https://www.gatsbyjs.com/blog/image-cdn-lightning-fast-image-processing-for-gatsby-cloud/)
+- [What is Image CDN?](https://support.gatsbyjs.com/hc/en-us/articles/4426379634835-What-is-Image-CDN-)
+- [How to enable Image CDN on Gatsby Cloud](https://support.gatsbyjs.com/hc/en-us/articles/4426393233171-How-to-Enable-Image-CDN)
+
+#### Querying for Gatsby Image CDN fields
+
+Follow [this guide](https://support.gatsbyjs.com/hc/en-us/articles/4426393233171-How-to-Enable-Image-CDN) to understand how to use the new `gatsbyImage` GraphQL field.
+
+#### Turning off file downloads
+
+When you're using Gatsby Image CDN you no longer need Gatsby to fetch all of the files in your Drupal instance. Turn that off with the following plugin option. This is required for Image CDN to work.
+
+```js
+  {
+    resolve: `gatsby-source-drupal`,
+    options: {
+      skipFileDownloads: true,
+      // other plugin options go here
+    },
+  },
+```
+
+#### Local dev improvements
+
+Using Image CDN also speeds up your local development startup times when running `gatsby develop`. Instead of fetching all files locally, `gatsby develop` has a local Image CDN emulator.
+This means Gatsby will only fetch and process the minimal amount of images required to render any page when you visit your Gatsby site at `http://localhost:8000`.
+
+#### Configuring placeholders for Gatsby Images
+
+By default full size images are fetched and scaled down to be used for low quality image placeholders (for lazy loading images on the frontend).
+This can make your builds slower than necessary so follow these steps to configure a new smaller placeholder image size in Drupal. This will speed up your builds when using Gatsby Image CDN.
+
+1. Install the [Consumer image styles module](https://www.drupal.org/project/consumer_image_styles)
+2. Navigate to "Extend->Web Services" and turn on "Consumer Image Styles" by checking the box and hitting save.
+3. Navigate to "Configuration->Image Styles". and add an image style called "Placeholder".
+4. Create a new scale effect and set its width and height to 20.
+5. If you already have a placeholder style you want to use, you can set the `gatsby-source-drupal` plugin option `placeholderStyleName` as the machine name of your style. \*\* See example option below
+6. For each entity that has an image field, navigate into "Configuration->Web Services->JSON:API->JSON:API Resource Overrides->Entity Type->(overwrite/edit)".
+7. Click on "advanced" for each image field you have, select "Image Styles (Image Field)" in the dropdown, then select the placeholder image style and save.
+8. Go to "Configuration->Web Services->Consumers" and add a default consumer if it doesn't already exist.
+9. Edit your default consumer and add the "Placeholder" image style by checking the box in the bottom section and saving.
+10. You may need to clear Drupal's cache under "Config->development->clear all caches".
+
+\*\* Example placeholder style plugin option
+
+```js
+{
+  resolve: `gatsby-source-drupal`,
+  options: {
+    placeholderStyleName: `custom_placeholder` // default is `placeholder`
+  }
+}
+```
+
 ### Filters
 
 You can use the `filters` option to limit the data that is retrieved from Drupal. Filters are applied per JSON API collection. You can use any [valid JSON API filter query](https://www.drupal.org/docs/8/modules/jsonapi/filtering). For large data sets this can reduce the build time of your application by allowing Gatsby to skip content you'll never use.

packages/gatsby-source-drupal/package.json

@@ -13,11 +13,13 @@
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.2",
     "fastq": "^1.13.0",
+    "gatsby-plugin-utils": "^3.7.0",
     "gatsby-source-filesystem": "^4.13.0",
     "got": "^11.8.3",
     "http2-wrapper": "^2.1.10",
     "lodash": "^4.17.21",
     "opentracing": "^0.14.7",
+    "probe-image-size": "^7.2.3",
     "tiny-async-pool": "^1.3.0",
     "url-join": "^4.0.1"
   },

packages/gatsby-source-drupal/src/__tests__/index.js

@@ -693,7 +693,7 @@ describe(`gatsby-source-drupal`, () => {
           { baseUrl }
         )
 
-        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(reporter.warn).toHaveBeenCalledTimes(2)
         expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
         expect(reporter.activityTimer).toHaveBeenNthCalledWith(
           1,

packages/gatsby-source-drupal/src/gatsby-node.js

@@ -6,11 +6,21 @@ import HttpAgent from "agentkeepalive"
 const opentracing = require(`opentracing`)
 const { SemanticAttributes } = require(`@opentelemetry/semantic-conventions`)
 
+const {
+  polyfillImageServiceDevRoutes,
+  addRemoteFilePolyfillInterface,
+} = require(`gatsby-plugin-utils/polyfill-remote-file`)
+
 const { HttpsAgent } = HttpAgent
 
 const { setOptions, getOptions } = require(`./plugin-options`)
 
-const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
+const {
+  nodeFromData,
+  downloadFile,
+  isFileNode,
+  imageCDNState,
+} = require(`./normalize`)
 const {
   initRefsLookups,
   storeRefsLookups,
@@ -19,6 +29,7 @@ const {
   createNodeIfItDoesNotExist,
   handleDeletedNode,
   drupalCreateNodeManifest,
+  getExtendedFileNodeData,
 } = require(`./utils`)
 
 const agent = {
@@ -377,7 +388,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
               nodesToUpdate = [nodeSyncData.data]
             }
             for (const nodeToUpdate of nodesToUpdate) {
-              createNodeIfItDoesNotExist({
+              await createNodeIfItDoesNotExist({
                 nodeToUpdate,
                 actions,
                 createNodeId,
@@ -640,21 +651,47 @@ ${JSON.stringify(webhookBody, null, 4)}`
   createNodesSpan.setTag(`sourceNodes.fetch.type`, `full`)
 
   const nodes = new Map()
+  const fileNodesExtendedData = getExtendedFileNodeData(allData)
 
   // first pass - create basic nodes
-  _.each(allData, contentType => {
-    if (!contentType) return
-    _.each(contentType.data, datum => {
-      if (!datum) return
-      const node = nodeFromData(datum, createNodeId, entityReferenceRevisions)
+  for (const contentType of allData) {
+    if (!contentType) {
+      continue
+    }
+
+    await asyncPool(concurrentFileRequests, contentType.data, async datum => {
+      if (!datum) {
+        return
+      }
+
+      const node = await nodeFromData(
+        datum,
+        createNodeId,
+        entityReferenceRevisions,
+        pluginOptions,
+        fileNodesExtendedData,
+        reporter
+      )
+
       drupalCreateNodeManifest({
         attributes: datum?.attributes,
         gatsbyNode: node,
         unstable_createNodeManifest,
       })
+
       nodes.set(node.id, node)
     })
-  })
+  }
+
+  if (
+    !imageCDNState.foundPlaceholderStyle &&
+    !imageCDNState.hasLoggedNoPlaceholderStyle
+  ) {
+    imageCDNState.hasLoggedNoPlaceholderStyle = true
+    reporter.warn(
+      `[gatsby-source-drupal]\nNo Gatsby Image CDN placeholder style found. Please ensure that you have a placeholder style in your Drupal site for the fastest builds. See the docs for more info on gatsby-source-drupal Image CDN support:\n\nhttps://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme`
+    )
+  }
 
   createNodesSpan.setTag(`sourceNodes.createNodes.count`, nodes.size)
 
@@ -665,6 +702,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
       mutateNode: true,
       createNodeId,
       entityReferenceRevisions,
+      fileNodesExtendedData,
     })
   })
 
@@ -716,7 +754,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
   createNodesSpan.finish()
   await storeRefsLookups({ cache, getNodes })
-  return
 }
 
 // This is maintained for legacy reasons and will eventually be removed.
@@ -813,3 +850,26 @@ exports.pluginOptionsSchema = ({ Joi }) =>
       nonTranslatableEntities: Joi.array().items(Joi.string()).required(),
     }),
   })
+
+exports.onCreateDevServer = async ({ app }) => {
+  // this makes the gatsby develop image CDN emulator work on earlier versions of Gatsby.
+  polyfillImageServiceDevRoutes(app)
+}
+
+exports.createSchemaCustomization = ({ actions, schema }) => {
+  actions.createTypes([
+    // polyfill so image CDN works on older versions of Gatsby
+    addRemoteFilePolyfillInterface(
+      // this type is merged in with the inferred file__file type, adding Image CDN support via the gatsbyImage GraphQL field. The `RemoteFile` interface as well as the polyfill above are what add the gatsbyImage field.
+      schema.buildObjectType({
+        name: `file__file`,
+        fields: {},
+        interfaces: [`Node`, `RemoteFile`],
+      }),
+      {
+        schema,
+        actions,
+      }
+    ),
+  ])
+}