feat(gatsby-source-filesystem): add createFileNodeFromBuffer (#14576)

Author: superhawk610

packages/gatsby-source-filesystem/README.md

@@ -100,10 +100,11 @@ To filter by the `name` you specified in the config, use `sourceInstanceName`:
 
 ## Helper functions
 
-`gatsby-source-filesystem` exports two helper functions:
+`gatsby-source-filesystem` exports three helper functions:
 
 - `createFilePath`
 - `createRemoteFileNode`
+- `createFileNodeFromBuffer`
 
 ### createFilePath
 
@@ -258,3 +259,90 @@ createRemoteFileNode({
   name: "image",
 })
 ```
+
+### createFileNodeFromBuffer
+
+When working with data that isn't already stored in a file, such as when querying binary/blob fields from a database, it's helpful to cache that data to the filesystem in order to use it with other transformers that accept files as input.
+
+The `createFileNodeFromBuffer` helper accepts a `Buffer`, caches its contents to disk, and creates a file node that points to it.
+
+## Example usage
+
+The following example is adapted from the source of [`gatsby-source-mysql`](https://github.com/malcolm-kee/gatsby-source-mysql):
+
+```js
+// gatsby-node.js
+const createMySqlNodes = require(`./create-nodes`)
+
+exports.sourceNodes = async (
+  { actions, createNodeId, store, cache },
+  config
+) => {
+  const { createNode } = actions
+  const { conn, queries } = config
+  const { db, results } = await query(conn, queries)
+
+  try {
+    queries
+      .map((query, i) => ({ ...query, ___sql: results[i] }))
+      .forEach(result =>
+        createMySqlNodes(result, results, createNode, {
+          createNode,
+          createNodeId,
+          store,
+          cache,
+        })
+      )
+    db.end()
+  } catch (e) {
+    console.error(e)
+    db.end()
+  }
+}
+
+// create-nodes.js
+const { createFileNodeFromBuffer } = require(`gatsby-source-filesystem`)
+const createNodeHelpers = require(`gatsby-node-helpers`).default
+
+const { createNodeFactory } = createNodeHelpers({ typePrefix: `mysql` })
+
+function attach(node, key, value, ctx) {
+  if (Buffer.isBuffer(value)) {
+    ctx.linkChildren.push(parentNodeId =>
+      createFileNodeFromBuffer({
+        buffer: value,
+        store: ctx.store,
+        cache: ctx.cache,
+        createNode: ctx.createNode,
+        createNodeId: ctx.createNodeId,
+      })
+    )
+    value = `Buffer`
+  }
+
+  node[key] = value
+}
+
+function createMySqlNodes({ name, __sql, idField, keys }, results, ctx) {
+  const MySqlNode = createNodeFactory(name)
+  ctx.linkChildren = []
+
+  return __sql.forEach(row => {
+    if (!keys) keys = Object.keys(row)
+
+    const node = { id: row[idField] }
+
+    for (const key of keys) {
+      attach(node, key, row[key], ctx)
+    }
+
+    node = ctx.createNode(node)
+
+    for (const link of ctx.linkChildren) {
+      link(node.id)
+    }
+  })
+}
+
+module.exports = createMySqlNodes
+```

packages/gatsby-source-filesystem/src/__tests__/create-file-node-from-buffer.js

@@ -0,0 +1,167 @@
+jest.mock(`fs-extra`, () => {
+  return {
+    ensureDir: jest.fn(() => true),
+    writeFile: jest.fn((_f, _b, cb) => cb()),
+    stat: jest.fn(() => {
+      return {
+        isDirectory: jest.fn(),
+      }
+    }),
+  }
+})
+jest.mock(`../create-file-node`, () => {
+  return {
+    createFileNode: jest.fn(() => {
+      return { internal: {} }
+    }),
+  }
+})
+
+const { ensureDir, writeFile } = require(`fs-extra`)
+const { createFileNode } = require(`../create-file-node`)
+const createFileNodeFromBuffer = require(`../create-file-node-from-buffer`)
+
+const createMockBuffer = content => {
+  const buffer = Buffer.alloc(content.length)
+  buffer.write(content)
+  return buffer
+}
+
+const createMockCache = () => {
+  return {
+    get: jest.fn(),
+    set: jest.fn(),
+  }
+}
+
+const bufferEq = (b1, b2) => Buffer.compare(b1, b2) === 0
+
+describe(`create-file-node-from-buffer`, () => {
+  const defaultArgs = {
+    store: {
+      getState: jest.fn(() => {
+        return {
+          program: {
+            directory: `__whatever__`,
+          },
+        }
+      }),
+    },
+    createNode: jest.fn(),
+    createNodeId: jest.fn(),
+  }
+
+  describe(`functionality`, () => {
+    afterEach(() => jest.clearAllMocks())
+
+    const setup = ({
+      hash,
+      buffer = createMockBuffer(`some binary content`),
+      cache = createMockCache(),
+    } = {}) =>
+      createFileNodeFromBuffer({
+        ...defaultArgs,
+        buffer,
+        hash,
+        cache,
+      })
+
+    it(`rejects when the buffer can't be read`, () => {
+      expect(setup({ buffer: null })).rejects.toEqual(
+        expect.stringContaining(`bad buffer`)
+      )
+    })
+
+    it(`caches the buffer's content locally`, async () => {
+      expect.assertions(2)
+
+      let output
+      writeFile.mockImplementationOnce((_f, buf, cb) => {
+        output = buf
+        cb()
+      })
+
+      const buffer = createMockBuffer(`buffer-content`)
+      await setup({ buffer })
+
+      expect(ensureDir).toBeCalledTimes(2)
+      expect(bufferEq(buffer, output)).toBe(true)
+    })
+
+    it(`uses cached file Promise for buffer with a matching hash`, async () => {
+      expect.assertions(3)
+
+      const cache = createMockCache()
+
+      await setup({ cache, hash: `same-hash` })
+      await setup({ cache, hash: `same-hash` })
+
+      expect(cache.get).toBeCalledTimes(1)
+      expect(cache.set).toBeCalledTimes(1)
+      expect(writeFile).toBeCalledTimes(1)
+    })
+
+    it(`uses cached file from previous run with a matching hash`, async () => {
+      expect.assertions(3)
+
+      const cache = createMockCache()
+      cache.get.mockImplementationOnce(() => `cached-file-path`)
+
+      await setup({ cache, hash: `cached-hash` })
+
+      expect(cache.get).toBeCalledWith(expect.stringContaining(`cached-hash`))
+      expect(cache.set).not.toBeCalled()
+      expect(createFileNode).toBeCalledWith(
+        expect.stringContaining(`cached-file-path`),
+        expect.any(Function),
+        expect.any(Object)
+      )
+    })
+  })
+
+  describe(`validation`, () => {
+    it(`throws on invalid inputs: createNode`, () => {
+      expect(() => {
+        createFileNodeFromBuffer({
+          ...defaultArgs,
+          createNode: undefined,
+        })
+      }).toThrowErrorMatchingInlineSnapshot(
+        `"createNode must be a function, was undefined"`
+      )
+    })
+
+    it(`throws on invalid inputs: createNodeId`, () => {
+      expect(() => {
+        createFileNodeFromBuffer({
+          ...defaultArgs,
+          createNodeId: undefined,
+        })
+      }).toThrowErrorMatchingInlineSnapshot(
+        `"createNodeId must be a function, was undefined"`
+      )
+    })
+
+    it(`throws on invalid inputs: cache`, () => {
+      expect(() => {
+        createFileNodeFromBuffer({
+          ...defaultArgs,
+          cache: undefined,
+        })
+      }).toThrowErrorMatchingInlineSnapshot(
+        `"cache must be the Gatsby cache, was undefined"`
+      )
+    })
+
+    it(`throws on invalid inputs: store`, () => {
+      expect(() => {
+        createFileNodeFromBuffer({
+          ...defaultArgs,
+          store: undefined,
+        })
+      }).toThrowErrorMatchingInlineSnapshot(
+        `"store must be the redux store, was undefined"`
+      )
+    })
+  })
+})