[perf] skip loading client manifest for static metadata routes (#77260)

### What

During development we had a retry logic to load the client manifest for
App Router routes, which is the requirement since #74835. But for static
metadata routes like `favicon.ico` it doesn't generate client manifest,
loading it will cause extra loading time.

Refactoring the metadata helpers to determine the static metadata routes
more strictly, mostly only cover the image one as their pathname and
entry filename is pretty consistent. For sitemap or robots.txt it's
still fine to leave as it in dev for now. The delay is mostly waiting
for manifest.

Long term goal is to build the static metadata routes as static output
rather than compile each time.

Author: huozhi

packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -18,7 +18,7 @@ import picomatch from 'next/dist/compiled/picomatch'
 import { getModuleBuildInfo } from '../loaders/get-module-build-info'
 import { getPageFilePath } from '../../entries'
 import { resolveExternal } from '../../handle-externals'
-import { isStaticMetadataRoute } from '../../../lib/metadata/is-metadata-route'
+import { isStaticMetadataRoutePage } from '../../../lib/metadata/is-metadata-route'
 import { getCompilationSpan } from '../utils'
 
 const PLUGIN_NAME = 'TraceEntryPointsPlugin'
@@ -243,7 +243,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
 
           const entryIsStaticMetadataRoute =
             appDirRelativeEntryPath &&
-            isStaticMetadataRoute(appDirRelativeEntryPath)
+            isStaticMetadataRoutePage(appDirRelativeEntryPath)
 
           // Include the client reference manifest in the trace, but not for
           // static metadata routes, for which we don't generate those.

packages/next/src/lib/metadata/is-metadata-route.ts

@@ -1,5 +1,6 @@
 import type { PageExtensions } from '../../build/page-extensions-type'
 import { normalizePathSep } from '../../shared/lib/page-path/normalize-path-sep'
+import { isAppRouteRoute } from '../is-app-route-route'
 
 export const STATIC_METADATA_IMAGES = {
   icon: {
@@ -127,15 +128,24 @@ export function isMetadataRouteFile(
   )
 }
 
-export function isStaticMetadataRouteFile(appDirRelativePath: string) {
+export function isStaticMetadataRoutePage(appDirRelativePath: string) {
   return isMetadataRouteFile(appDirRelativePath, [], true)
 }
 
-export function isStaticMetadataRoute(page: string) {
+// Check if the route is a static metadata route, with /route suffix
+// e.g. /favicon.ico/route, /icon.png/route, etc.
+// But skip the text routes like robots.txt since they might also be dynamic.
+// Checking route path is not enough to determine if text routes is dynamic.
+export function isStaticMetadataRoute(route: string) {
+  const pathname = route.slice(0, -'/route'.length)
   return (
-    page === '/robots' ||
-    page === '/manifest' ||
-    isStaticMetadataRouteFile(page)
+    isAppRouteRoute(route) &&
+    isStaticMetadataRoutePage(pathname) &&
+    // These routes can either be built by static or dynamic entrypoints,
+    // so we assume they're dynamic
+    pathname !== '/robots.txt' &&
+    pathname !== '/manifest.webmanifest' &&
+    !pathname.endsWith('/sitemap.xml')
   )
 }
 

packages/next/src/server/load-components.ts

@@ -33,6 +33,7 @@ import { setReferenceManifestsSingleton } from './app-render/encryption-utils'
 import { createServerModuleMap } from './app-render/action-utils'
 import type { DeepReadonly } from '../shared/lib/deep-readonly'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
+import { isStaticMetadataRoute } from '../lib/metadata/is-metadata-route'
 
 export type ManifestItem = {
   id: number | string
@@ -195,6 +196,9 @@ async function loadComponentsImpl<N = any>({
     )
   }
 
+  // Make sure to avoid loading the manifest for static metadata routes for better performance.
+  const hasClientManifest = !isStaticMetadataRoute(page)
+
   // Load the manifest files first
   //
   // Loading page-specific manifests shouldn't throw an error if the manifest couldn't be found, so
@@ -223,7 +227,7 @@ async function loadComponentsImpl<N = any>({
           join(distDir, `${DYNAMIC_CSS_MANIFEST}.json`),
           manifestLoadAttempts
         ).catch(() => undefined),
-    isAppPath
+    isAppPath && hasClientManifest
       ? tryLoadClientReferenceManifest(
           join(
             distDir,

packages/next/src/server/route-matcher-providers/dev/dev-app-route-route-matcher-provider.ts

@@ -53,7 +53,7 @@ export class DevAppRouteRouteMatcherProvider extends FileCacheRouteMatcherProvid
         true
       )
 
-      if (!isStaticMetadataRoute(page) && isEntryMetadataRouteFile) {
+      if (isEntryMetadataRouteFile && !isStaticMetadataRoute(page)) {
         // Matching dynamic metadata routes.
         // Add 2 possibilities for both single and multiple routes:
         {