Move Pages render out of server (#51678)

This removes calls for rendering files in `pages/` via the server code and relies instead relies on the bundled `routeModule` to perform the rendering.

Future cases that try to call the legacy render will hit a new: 

```
Invariant: render should have used routeModule
```

Error. These cases should be reported.

Author: wyattjoh

packages/next/src/build/entries.ts

@@ -53,7 +53,7 @@ import { isAppRouteRoute } from '../lib/is-app-route-route'
 import { normalizeMetadataRoute } from '../lib/metadata/get-metadata-route'
 import { fileExists } from '../lib/file-exists'
 import { getRouteLoaderEntry } from './webpack/loaders/next-route-loader'
-import { isInternalPathname } from '../lib/is-internal-pathname'
+import { isInternalComponent } from '../lib/is-internal-component'
 import { isStaticMetadataRouteFile } from '../lib/metadata/is-metadata-route'
 
 export async function getStaticInfoIncludingLayouts({
@@ -596,16 +596,15 @@ export async function createEntrypoints(
           } else if (
             !isAPIRoute(page) &&
             !isMiddlewareFile(page) &&
-            !isInternalPathname(absolutePagePath)
+            !isInternalComponent(absolutePagePath)
           ) {
             server[serverBundlePath] = [
               getRouteLoaderEntry({
                 page,
+                pages,
                 absolutePagePath,
                 preferredRegion: staticInfo.preferredRegion,
-                middlewareConfig: Buffer.from(
-                  JSON.stringify(staticInfo.middleware || {})
-                ).toString('base64'),
+                middlewareConfig: staticInfo.middleware ?? {},
               }),
             ]
           } else {

packages/next/src/build/webpack/loaders/next-route-loader/index.ts

@@ -6,6 +6,16 @@ import { PagesRouteModuleOptions } from '../../../../server/future/route-modules
 import { RouteKind } from '../../../../server/future/route-kind'
 import { normalizePagePath } from '../../../../shared/lib/page-path/normalize-page-path'
 import { MiddlewareConfig } from '../../../analysis/get-page-static-info'
+import { decodeFromBase64, encodeToBase64 } from '../utils'
+import { isInstrumentationHookFile } from '../../../worker'
+
+type RouteLoaderOptionsInput = {
+  page: string
+  pages: { [page: string]: string }
+  preferredRegion: string | string[] | undefined
+  absolutePagePath: string
+  middlewareConfig: MiddlewareConfig
+}
 
 /**
  * The options for the route loader.
@@ -25,17 +35,29 @@ type RouteLoaderOptions = {
    * The absolute path to the userland page file.
    */
   absolutePagePath: string
-
-  middlewareConfig: string
+  absoluteAppPath: string
+  absoluteDocumentPath: string
+  middlewareConfigBase64: string
 }
 
 /**
  * Returns the loader entry for a given page.
  *
- * @param query the options to create the loader entry
+ * @param options the options to create the loader entry
  * @returns the encoded loader entry
  */
-export function getRouteLoaderEntry(query: RouteLoaderOptions): string {
+export function getRouteLoaderEntry(options: RouteLoaderOptionsInput): string {
+  const query: RouteLoaderOptions = {
+    page: options.page,
+    preferredRegion: options.preferredRegion,
+    absolutePagePath: options.absolutePagePath,
+    // These are the path references to the internal components that may be
+    // overridden by userland components.
+    absoluteAppPath: options.pages['/_app'],
+    absoluteDocumentPath: options.pages['/_document'],
+    middlewareConfigBase64: encodeToBase64(options.middlewareConfig),
+  }
+
   return `next-route-loader?${stringify(query)}!`
 }
 
@@ -49,16 +71,18 @@ const loader: webpack.LoaderDefinitionFunction<RouteLoaderOptions> =
       page,
       preferredRegion,
       absolutePagePath,
-      middlewareConfig: middlewareConfigBase64,
+      absoluteAppPath,
+      absoluteDocumentPath,
+      middlewareConfigBase64,
     } = this.getOptions()
 
     // Ensure we only run this loader for as a module.
     if (!this._module) {
       throw new Error('Invariant: expected this to reference a module')
     }
 
-    const middlewareConfig: MiddlewareConfig = JSON.parse(
-      Buffer.from(middlewareConfigBase64, 'base64').toString()
+    const middlewareConfig: MiddlewareConfig = decodeFromBase64(
+      middlewareConfigBase64
     )
 
     // Attach build info to the module.
@@ -70,7 +94,7 @@ const loader: webpack.LoaderDefinitionFunction<RouteLoaderOptions> =
       middlewareConfig,
     }
 
-    const options: Omit<PagesRouteModuleOptions, 'userland'> = {
+    const options: Omit<PagesRouteModuleOptions, 'userland' | 'components'> = {
       definition: {
         kind: RouteKind.PAGES,
         page: normalizePagePath(page),
@@ -86,6 +110,10 @@ const loader: webpack.LoaderDefinitionFunction<RouteLoaderOptions> =
         import RouteModule from "next/dist/server/future/route-modules/pages/module"
         import { hoist } from "next/dist/build/webpack/loaders/next-route-loader/helpers"
 
+        // Import the app and document modules.
+        import * as moduleDocument from ${JSON.stringify(absoluteDocumentPath)}
+        import * as moduleApp from ${JSON.stringify(absoluteAppPath)}
+
         // Import the userland code.
         import * as userland from ${JSON.stringify(absolutePagePath)}
 
@@ -98,6 +126,14 @@ const loader: webpack.LoaderDefinitionFunction<RouteLoaderOptions> =
         export const getServerSideProps = hoist(userland, "getServerSideProps")
         export const config = hoist(userland, "config")
         export const reportWebVitals = hoist(userland, "reportWebVitals")
+        ${
+          // When we're building the instrumentation page (only when the
+          // instrumentation file conflicts with a page also labeled
+          // /instrumentation) hoist the `register` method.
+          isInstrumentationHookFile(page)
+            ? 'export const register = hoist(userland, "register")'
+            : ''
+        }
 
         // Re-export legacy methods.
         export const unstable_getStaticProps = hoist(userland, "unstable_getStaticProps")
@@ -108,7 +144,14 @@ const loader: webpack.LoaderDefinitionFunction<RouteLoaderOptions> =
 
         // Create and export the route module that will be consumed.
         const options = ${JSON.stringify(options)}
-        const routeModule = new RouteModule({ ...options, userland })
+        const routeModule = new RouteModule({
+          ...options,
+          components: {
+            App: moduleApp.default,
+            Document: moduleDocument.default,
+          },
+          userland,
+        })
         
         export { routeModule }
     `

packages/next/src/build/webpack/loaders/utils.ts

@@ -50,3 +50,11 @@ export function generateActionId(filePath: string, exportName: string) {
     .update(filePath + ':' + exportName)
     .digest('hex')
 }
+
+export function encodeToBase64<T extends {}>(obj: T): string {
+  return Buffer.from(JSON.stringify(obj)).toString('base64')
+}
+
+export function decodeFromBase64<T extends {}>(str: string): T {
+  return JSON.parse(Buffer.from(str, 'base64').toString('utf8'))
+}

packages/next/src/lib/is-internal-component.ts

@@ -0,0 +1,9 @@
+export function isInternalComponent(pathname: string): boolean {
+  switch (pathname) {
+    case 'next/dist/pages/_app':
+    case 'next/dist/pages/_document':
+      return true
+    default:
+      return false
+  }
+}

packages/next/src/lib/is-internal-pathname.ts

@@ -30,6 +30,7 @@ import type { NextFontManifest } from '../build/webpack/plugins/next-font-manife
 import type { PagesRouteModule } from './future/route-modules/pages/module'
 import type { NodeNextRequest, NodeNextResponse } from './base-http/node'
 import type { AppRouteRouteMatch } from './future/route-matches/app-route-route-match'
+import type { RouteDefinition } from './future/route-definitions/route-definition'
 
 import { format as formatUrl, parse as parseUrl } from 'url'
 import { getRedirectStatus } from '../lib/redirect-status'
@@ -109,7 +110,11 @@ import {
   toNodeOutgoingHttpHeaders,
 } from './web/utils'
 import { NEXT_QUERY_PARAM_PREFIX } from '../lib/constants'
-import { isRouteMatch } from './future/route-matches/route-match'
+import {
+  isRouteMatch,
+  parsedUrlQueryToParams,
+  type RouteMatch,
+} from './future/route-matches/route-match'
 
 export type FindComponentsResult = {
   components: LoadComponentsReturnType
@@ -1675,7 +1680,18 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       let result: RenderResult
 
       // Get the match for the page if it exists.
-      const match = getRequestMeta(req, '_nextMatch')
+      const match: RouteMatch<RouteDefinition<RouteKind>> | undefined =
+        getRequestMeta(req, '_nextMatch') ??
+        // If the match can't be found, rely on the loaded route module. This
+        // should only be required during development when we add FS routes.
+        (this.renderOpts.dev && components.ComponentMod?.routeModule
+          ? {
+              definition: components.ComponentMod.routeModule.definition,
+              params: opts.params
+                ? parsedUrlQueryToParams(opts.params)
+                : undefined,
+            }
+          : undefined)
 
       if (
         match &&
@@ -2578,6 +2594,13 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       const fallbackComponents = await this.getFallbackErrorComponents()
 
       if (fallbackComponents) {
+        // There was an error, so use it's definition from the route module
+        // to add the match to the request.
+        addRequestMeta(ctx.req, '_nextMatch', {
+          definition: fallbackComponents.ComponentMod.routeModule.definition,
+          params: undefined,
+        })
+
         return this.renderToResponseWithComponents(
           {
             ...ctx,

packages/next/src/server/base-server.ts

@@ -47,7 +47,6 @@ import getRouteFromEntrypoint from '../get-route-from-entrypoint'
 import { fileExists } from '../../lib/file-exists'
 import {
   difference,
-  isInstrumentationHookFile,
   isMiddlewareFile,
   isMiddlewareFilename,
 } from '../../build/utils'
@@ -62,7 +61,7 @@ import { RouteMatch } from '../future/route-matches/route-match'
 import { parseVersionInfo, VersionInfo } from './parse-version-info'
 import { isAPIRoute } from '../../lib/is-api-route'
 import { getRouteLoaderEntry } from '../../build/webpack/loaders/next-route-loader'
-import { isInternalPathname } from '../../lib/is-internal-pathname'
+import { isInternalComponent } from '../../lib/is-internal-component'
 
 function diff(a: Set<any>, b: Set<any>) {
   return new Set([...a].filter((v) => !b.has(v)))
@@ -899,16 +898,14 @@ export default class HotReloader {
                 } else if (
                   !isAPIRoute(page) &&
                   !isMiddlewareFile(page) &&
-                  !isInternalPathname(relativeRequest) &&
-                  !isInstrumentationHookFile(page)
+                  !isInternalComponent(relativeRequest)
                 ) {
                   value = getRouteLoaderEntry({
                     page,
+                    pages: this.pagesMapping,
                     absolutePagePath: relativeRequest,
                     preferredRegion: staticInfo.preferredRegion,
-                    middlewareConfig: Buffer.from(
-                      JSON.stringify(staticInfo.middleware || {})
-                    ).toString('base64'),
+                    middlewareConfig: staticInfo.middleware ?? {},
                   })
                 } else {
                   value = relativeRequest

packages/next/src/server/dev/hot-reloader.ts

@@ -1,3 +1,4 @@
+import type { ParsedUrlQuery } from 'querystring'
 import type { RouteDefinition } from '../route-definitions/route-definition'
 
 /**
@@ -31,3 +32,22 @@ export function isRouteMatch<M extends RouteMatch>(
 ): match is M {
   return match.definition.kind === kind
 }
+
+/**
+ * Converts the query into params.
+ *
+ * @param query the query to convert to params
+ * @returns the params
+ */
+export function parsedUrlQueryToParams(
+  query: ParsedUrlQuery
+): Record<string, string | string[]> {
+  const params: Record<string, string | string[]> = {}
+
+  for (const [key, value] of Object.entries(query)) {
+    if (typeof value === 'undefined') continue
+    params[key] = value
+  }
+
+  return params
+}

packages/next/src/server/future/route-matches/route-match.ts

@@ -0,0 +1,23 @@
+import Document from '../../../../../pages/_document'
+import App from '../../../../../pages/_app'
+import { RouteKind } from '../../../route-kind'
+
+import * as moduleError from '../../../../../pages/_error'
+
+import PagesRouteModule from '../module'
+
+export const routeModule = new PagesRouteModule({
+  // TODO: add descriptor for internal error page
+  definition: {
+    kind: RouteKind.PAGES,
+    page: '/_error',
+    pathname: '/_error',
+    filename: '',
+    bundlePath: '',
+  },
+  components: {
+    App,
+    Document,
+  },
+  userland: moduleError,
+})