Add partial support for "use cache" in metadata route handlers (#74835)

Adds support for using `"use cache"` in the special metadata route
handlers like
[`sitemap.ts`](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/sitemap#generating-a-sitemap-using-code-js-ts),
[`opengraph-image.tsx`](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/opengraph-image#generate-images-using-code-js-ts-tsx),
[`icon.tsx`](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/app-icons#generate-icons-using-code-js-ts-tsx),
and other [metadata
files](https://nextjs.org/docs/app/api-reference/file-conventions/metadata).

reverts #71225
fixes #74146
closes NAR-51

As a follow-up we need to ensure that opengraph image responses do not
bail out of static generation when `dynamicIO` is enabled.

Author: unstubbable

Diff for: crates/next-api/src/app.rs

@@ -1309,6 +1309,17 @@ export default async function getBaseWebpackConfig(
                     },
                   ],
                 },
+                resourceQuery: {
+                  // Do not apply next-flight-loader to imports generated by the
+                  // next-metadata-image-loader, to avoid generating unnecessary
+                  // and conflicting entries in the flight client entry plugin.
+                  // These are already covered by the next-metadata-route-loader
+                  // entries.
+                  not: [
+                    new RegExp(WEBPACK_RESOURCE_QUERIES.metadata),
+                    new RegExp(WEBPACK_RESOURCE_QUERIES.metadataImageMeta),
+                  ],
+                },
                 resolve: {
                   mainFields: getMainField(compilerType, true),
                   conditionNames: reactServerCondition,

Diff for: packages/next/src/build/webpack-config.ts

@@ -4,7 +4,7 @@ import type {
 } from '../loaders/next-flight-client-entry-loader'
 
 import { webpack } from 'next/dist/compiled/webpack/webpack'
-import { stringify } from 'querystring'
+import { parse, stringify } from 'querystring'
 import path from 'path'
 import { sources } from 'next/dist/compiled/webpack/webpack'
 import {
@@ -13,7 +13,10 @@ import {
   EntryTypes,
   getEntryKey,
 } from '../../../server/dev/on-demand-entry-handler'
-import { WEBPACK_LAYERS } from '../../../lib/constants'
+import {
+  WEBPACK_LAYERS,
+  WEBPACK_RESOURCE_QUERIES,
+} from '../../../lib/constants'
 import {
   APP_CLIENT_INTERNALS,
   BARREL_OPTIMIZATION_PREFIX,
@@ -41,6 +44,7 @@ import { PAGE_TYPES } from '../../../lib/page-types'
 import { getModuleBuildInfo } from '../loaders/get-module-build-info'
 import { getAssumedSourceType } from '../loaders/next-flight-loader'
 import { isAppRouteRoute } from '../../../lib/is-app-route-route'
+import { isMetadataRoute } from '../../../lib/metadata/is-metadata-route'
 
 interface Options {
   dev: boolean
@@ -296,10 +300,14 @@ export class FlightClientEntryPlugin {
         compilation.moduleGraph
       )) {
         // Entry can be any user defined entry files such as layout, page, error, loading, etc.
-        const entryRequest = (
+        let entryRequest = (
           connection.dependency as unknown as webpack.NormalModule
         ).request
 
+        if (entryRequest.endsWith(WEBPACK_RESOURCE_QUERIES.metadataRoute)) {
+          entryRequest = getMetadataRouteResource(entryRequest)
+        }
+
         const { clientComponentImports, actionImports, cssImports } =
           this.collectComponentInfoFromServerEntryDependency({
             entryRequest,
@@ -332,10 +340,16 @@ export class FlightClientEntryPlugin {
           : entryRequest
 
         // Replace file suffix as `.js` will be added.
-        const bundlePath = normalizePathSep(
+        let bundlePath = normalizePathSep(
           relativeRequest.replace(/\.[^.\\/]+$/, '').replace(/^src[\\/]/, '')
         )
 
+        // For metadata routes, the entry name can be used as the bundle path,
+        // as it has been normalized already.
+        if (isMetadataRoute(bundlePath)) {
+          bundlePath = name
+        }
+
         Object.assign(mergedCSSimports, cssImports)
         clientEntriesToInject.push({
           compiler,
@@ -1094,5 +1108,16 @@ function getModuleResource(mod: webpack.NormalModule): string {
   if (mod.matchResource?.startsWith(BARREL_OPTIMIZATION_PREFIX)) {
     modResource = mod.matchResource + ':' + modResource
   }
+
+  if (mod.resource === `?${WEBPACK_RESOURCE_QUERIES.metadataRoute}`) {
+    return getMetadataRouteResource(mod.rawRequest)
+  }
+
   return modResource
 }
+
+function getMetadataRouteResource(request: string): string {
+  const query = request.split('next-metadata-route-loader?')[1]
+
+  return parse(query).filePath as string
+}

Diff for: packages/next/src/build/webpack/plugins/flight-client-entry-plugin.ts

@@ -26,7 +26,6 @@ import {
 } from '../utils'
 import type { ChunkGroup } from 'webpack'
 import { encodeURIPath } from '../../../shared/lib/encode-uri-path'
-import { isMetadataRoute } from '../../../lib/metadata/is-metadata-route'
 import type { ModuleInfo } from './flight-client-entry-plugin'
 
 interface Options {
@@ -559,9 +558,9 @@ export class ClientReferenceManifestPlugin {
         manifestEntryFiles.push(entryName.replace(/\/page(\.[^/]+)?$/, '/page'))
       }
 
-      // We also need to create manifests for route handler entrypoints
-      // (excluding metadata route handlers) to enable `'use cache'`.
-      if (/\/route$/.test(entryName) && !isMetadataRoute(entryName)) {
+      // We also need to create manifests for route handler entrypoints to
+      // enable `'use cache'`.
+      if (/\/route$/.test(entryName)) {
         manifestEntryFiles.push(entryName)
       }
 

Diff for: packages/next/src/build/webpack/plugins/flight-manifest-plugin.ts

@@ -19,7 +19,6 @@ import picomatch from 'next/dist/compiled/picomatch'
 import { getModuleBuildInfo } from '../loaders/get-module-build-info'
 import { getPageFilePath } from '../../entries'
 import { resolveExternal } from '../../handle-externals'
-import { isMetadataRoute } from '../../../lib/metadata/is-metadata-route'
 
 const PLUGIN_NAME = 'TraceEntryPointsPlugin'
 export const TRACE_IGNORES = [
@@ -243,18 +242,15 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
         )
 
         if (entrypoint.name.startsWith('app/')) {
-          // Include the client reference manifest for pages and route handlers,
-          // excluding metadata route handlers.
-          const clientManifestsForEntrypoint = isMetadataRoute(entrypoint.name)
-            ? null
-            : nodePath.join(
-                outputPath,
-                outputPrefix,
-                entrypoint.name.replace(/%5F/g, '_') +
-                  '_' +
-                  CLIENT_REFERENCE_MANIFEST +
-                  '.js'
-              )
+          // include the client reference manifest
+          const clientManifestsForEntrypoint = nodePath.join(
+            outputPath,
+            outputPrefix,
+            entrypoint.name.replace(/%5F/g, '_') +
+              '_' +
+              CLIENT_REFERENCE_MANIFEST +
+              '.js'
+          )
 
           if (clientManifestsForEntrypoint !== null) {
             entryFiles.add(clientManifestsForEntrypoint)

Diff for: packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -7,7 +7,6 @@ import type {
   ModuleGraph,
 } from 'webpack'
 import type { ModuleGraphConnection } from 'webpack'
-import { isMetadataRoute } from '../../lib/metadata/is-metadata-route'
 
 export function traverseModules(
   compilation: Compilation,
@@ -48,11 +47,7 @@ export function forEachEntryModule(
 ) {
   for (const [name, entry] of compilation.entries.entries()) {
     // Skip for entries under pages/
-    if (
-      name.startsWith('pages/') ||
-      // Skip for metadata route handlers
-      (name.startsWith('app/') && isMetadataRoute(name))
-    ) {
+    if (name.startsWith('pages/')) {
       continue
     }
 

Diff for: packages/next/src/build/webpack/utils.ts

@@ -32,7 +32,6 @@ import { wait } from '../lib/wait'
 import { setReferenceManifestsSingleton } from './app-render/encryption-utils'
 import { createServerModuleMap } from './app-render/action-utils'
 import type { DeepReadonly } from '../shared/lib/deep-readonly'
-import { isMetadataRoute } from '../lib/metadata/is-metadata-route'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 
 export type ManifestItem = {
@@ -169,9 +168,6 @@ async function loadComponentsImpl<N = any>({
     ])
   }
 
-  // Make sure to avoid loading the manifest for metadata route handlers.
-  const hasClientManifest = isAppPath && !isMetadataRoute(page)
-
   // In dev mode we retry loading a manifest file to handle a race condition
   // that can occur while app and pages are compiling at the same time, and the
   // build-manifest is still being written to disk while an app path is
@@ -227,7 +223,7 @@ async function loadComponentsImpl<N = any>({
           join(distDir, `${DYNAMIC_CSS_MANIFEST}.json`),
           manifestLoadAttempts
         ).catch(() => undefined),
-    hasClientManifest
+    isAppPath
       ? tryLoadClientReferenceManifest(
           join(
             distDir,

Diff for: packages/next/src/server/load-components.ts

@@ -0,0 +1,38 @@
+import { ImageResponse } from 'next/og'
+import { setTimeout } from 'timers/promises'
+
+export const size = { width: 32, height: 32 }
+export const contentType = 'image/png'
+
+async function fetchIconLetter() {
+  'use cache'
+
+  // Simulate I/O
+  await setTimeout(100)
+
+  return 'N'
+}
+
+export default async function Icon() {
+  const letter = await fetchIconLetter()
+
+  return new ImageResponse(
+    (
+      <div
+        style={{
+          fontSize: 24,
+          background: 'black',
+          width: '100%',
+          height: '100%',
+          display: 'flex',
+          alignItems: 'center',
+          justifyContent: 'center',
+          color: 'white',
+        }}
+      >
+        {letter}
+      </div>
+    ),
+    { ...size }
+  )
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/icon.tsx

@@ -0,0 +1,12 @@
+import type { MetadataRoute } from 'next'
+import { getSentinelValue } from './sentinel'
+import { setTimeout } from 'timers/promises'
+
+export default async function manifest(): Promise<MetadataRoute.Manifest> {
+  'use cache'
+
+  // Simulate I/O
+  await setTimeout(100)
+
+  return { name: getSentinelValue() }
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/manifest.ts

@@ -0,0 +1,38 @@
+import { ImageResponse } from 'next/og'
+
+export const alt = 'About Acme'
+export const size = { width: 1200, height: 630 }
+export const contentType = 'image/png'
+
+async function fetchPostData() {
+  'use cache'
+
+  return { title: 'Test', created: Date.now() }
+}
+
+export default async function Image() {
+  const post = await fetchPostData()
+
+  return new ImageResponse(
+    (
+      <div
+        style={{
+          fontSize: 48,
+          background: 'white',
+          width: '100%',
+          height: '100%',
+          display: 'flex',
+          alignItems: 'center',
+          justifyContent: 'center',
+          flexDirection: 'column',
+        }}
+      >
+        <h1>{post.title}</h1>
+        <p style={{ fontSize: 32 }}>
+          {new Date(post.created).toLocaleTimeString()}
+        </p>
+      </div>
+    ),
+    size
+  )
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/opengraph-image.tsx

@@ -0,0 +1,20 @@
+import type { MetadataRoute } from 'next'
+import { getSentinelValue } from '../sentinel'
+import { setTimeout } from 'timers/promises'
+
+export async function generateSitemaps() {
+  return [{ id: 0 }, { id: 1 }]
+}
+
+export default async function sitemap({
+  id,
+}: {
+  id: number
+}): Promise<MetadataRoute.Sitemap> {
+  'use cache'
+
+  // Simulate I/O
+  await setTimeout(100)
+
+  return [{ url: `https://acme.com/${id}?sentinel=${getSentinelValue()}` }]
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/products/sitemap.ts

@@ -0,0 +1,14 @@
+import type { MetadataRoute } from 'next'
+import { getSentinelValue } from './sentinel'
+import { setTimeout } from 'timers/promises'
+
+export default async function robots(): Promise<MetadataRoute.Robots> {
+  'use cache'
+
+  // Simulate I/O
+  await setTimeout(100)
+
+  return {
+    rules: { userAgent: '*', allow: `/${getSentinelValue()}` },
+  }
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/robots.ts

@@ -0,0 +1,7 @@
+const { PHASE_PRODUCTION_BUILD } = require('next/constants')
+
+export function getSentinelValue() {
+  return process.env.NEXT_PHASE === PHASE_PRODUCTION_BUILD
+    ? 'buildtime'
+    : 'runtime'
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/sentinel.ts

@@ -0,0 +1,12 @@
+import type { MetadataRoute } from 'next'
+import { getSentinelValue } from './sentinel'
+import { setTimeout } from 'timers/promises'
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  'use cache'
+
+  // Simulate I/O
+  await setTimeout(100)
+
+  return [{ url: `https://acme.com?sentinel=${getSentinelValue()}` }]
+}

Diff for: test/e2e/app-dir/use-cache-metadata-route-handler/app/sitemap.ts

@@ -0,0 +1,10 @@
+/**
+ * @type {import('next').NextConfig}
+ */
+const nextConfig = {
+  experimental: {
+    dynamicIO: true,
+  },
+}
+
+module.exports = nextConfig