diff --git a/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts b/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
index 5c81244d24677..ab468725ae4a9 100644
--- a/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
+++ b/packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
@@ -18,7 +18,7 @@ import picomatch from 'next/dist/compiled/picomatch'
 import { getModuleBuildInfo } from '../loaders/get-module-build-info'
 import { getPageFilePath } from '../../entries'
 import { resolveExternal } from '../../handle-externals'
-import { isStaticMetadataRoute } from '../../../lib/metadata/is-metadata-route'
+import { isStaticMetadataRoutePage } from '../../../lib/metadata/is-metadata-route'
 import { getCompilationSpan } from '../utils'
 
 const PLUGIN_NAME = 'TraceEntryPointsPlugin'
@@ -243,7 +243,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
 
           const entryIsStaticMetadataRoute =
             appDirRelativeEntryPath &&
-            isStaticMetadataRoute(appDirRelativeEntryPath)
+            isStaticMetadataRoutePage(appDirRelativeEntryPath)
 
           // Include the client reference manifest in the trace, but not for
           // static metadata routes, for which we don't generate those.
diff --git a/packages/next/src/lib/metadata/is-metadata-route.ts b/packages/next/src/lib/metadata/is-metadata-route.ts
index 8e990c40d2d32..1d053fe4ae69a 100644
--- a/packages/next/src/lib/metadata/is-metadata-route.ts
+++ b/packages/next/src/lib/metadata/is-metadata-route.ts
@@ -1,5 +1,6 @@
 import type { PageExtensions } from '../../build/page-extensions-type'
 import { normalizePathSep } from '../../shared/lib/page-path/normalize-path-sep'
+import { isAppRouteRoute } from '../is-app-route-route'
 
 export const STATIC_METADATA_IMAGES = {
   icon: {
@@ -127,15 +128,24 @@ export function isMetadataRouteFile(
   )
 }
 
-export function isStaticMetadataRouteFile(appDirRelativePath: string) {
+export function isStaticMetadataRoutePage(appDirRelativePath: string) {
   return isMetadataRouteFile(appDirRelativePath, [], true)
 }
 
-export function isStaticMetadataRoute(page: string) {
+// Check if the route is a static metadata route, with /route suffix
+// e.g. /favicon.ico/route, /icon.png/route, etc.
+// But skip the text routes like robots.txt since they might also be dynamic.
+// Checking route path is not enough to determine if text routes is dynamic.
+export function isStaticMetadataRoute(route: string) {
+  const pathname = route.slice(0, -'/route'.length)
   return (
-    page === '/robots' ||
-    page === '/manifest' ||
-    isStaticMetadataRouteFile(page)
+    isAppRouteRoute(route) &&
+    isStaticMetadataRoutePage(pathname) &&
+    // These routes can either be built by static or dynamic entrypoints,
+    // so we assume they're dynamic
+    pathname !== '/robots.txt' &&
+    pathname !== '/manifest.webmanifest' &&
+    !pathname.endsWith('/sitemap.xml')
   )
 }
 
diff --git a/packages/next/src/server/load-components.ts b/packages/next/src/server/load-components.ts
index 40c6f6d670708..00cfba7f2c2ba 100644
--- a/packages/next/src/server/load-components.ts
+++ b/packages/next/src/server/load-components.ts
@@ -33,6 +33,7 @@ import { setReferenceManifestsSingleton } from './app-render/encryption-utils'
 import { createServerModuleMap } from './app-render/action-utils'
 import type { DeepReadonly } from '../shared/lib/deep-readonly'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
+import { isStaticMetadataRoute } from '../lib/metadata/is-metadata-route'
 
 export type ManifestItem = {
   id: number | string
@@ -195,6 +196,9 @@ async function loadComponentsImpl<N = any>({
     )
   }
 
+  // Make sure to avoid loading the manifest for static metadata routes for better performance.
+  const hasClientManifest = !isStaticMetadataRoute(page)
+
   // Load the manifest files first
   //
   // Loading page-specific manifests shouldn't throw an error if the manifest couldn't be found, so
@@ -223,7 +227,7 @@ async function loadComponentsImpl<N = any>({
           join(distDir, `${DYNAMIC_CSS_MANIFEST}.json`),
           manifestLoadAttempts
         ).catch(() => undefined),
-    isAppPath
+    isAppPath && hasClientManifest
       ? tryLoadClientReferenceManifest(
           join(
             distDir,
diff --git a/packages/next/src/server/route-matcher-providers/dev/dev-app-route-route-matcher-provider.ts b/packages/next/src/server/route-matcher-providers/dev/dev-app-route-route-matcher-provider.ts
index 72ba221e2fb1d..454f0ab896b15 100644
--- a/packages/next/src/server/route-matcher-providers/dev/dev-app-route-route-matcher-provider.ts
+++ b/packages/next/src/server/route-matcher-providers/dev/dev-app-route-route-matcher-provider.ts
@@ -53,7 +53,7 @@ export class DevAppRouteRouteMatcherProvider extends FileCacheRouteMatcherProvid
         true
       )
 
-      if (!isStaticMetadataRoute(page) && isEntryMetadataRouteFile) {
+      if (isEntryMetadataRouteFile && !isStaticMetadataRoute(page)) {
         // Matching dynamic metadata routes.
         // Add 2 possibilities for both single and multiple routes:
         {