[next-ts-plugin] fix: language service crashes / metadata plugin not working (#77213)

### Why?

This PR tackles two issues:

1. The Next.js TS plugin for metadata wasn't working.
2. The language service occasionally crashes in large projects.

#### The Next.js TS plugin for metadata wasn't working

The plugin wasn't working due to the position mismatch of
`optionalReplacementSpan` since VSCode
[v1.86](https://code.visualstudio.com/updates/v1_86) due to the behavior
change of mapping the `replacementSpan` with `optionalReplacementSpan`
if it exists.

x-ref:
https://github.com/microsoft/vscode/pull/200945/files#diff-9027dfcf905374dcada6a75144ed94935254a5a6bf325a20d557201cd7fbe704R766

#### The language service occasionally crashes in large projects

Since we spun up another language service and a custom language service
host for metadata rules, it occasionally crashed on large projects.

### How?

- Synced the `optionalReplacementSpan` to the new position.
- Removed creating another language service and relying on
`@typescript/vfs` for the virtual file system, which is recommended:
"where files on disk aren't the source of truth".


https://github.com/user-attachments/assets/b5b325a4-fd2f-4210-be68-1aa5e40229c2

### Success Criteria

- [x] Does it do completions with `export const metadata = { a }`?
- [x] Does it disable the plugin with `{ enabled: false }` in the
tsconfig.json?
- [x] Does the language service not break when used in large projects?

### Follow Up

This fix is a partial port from the commit
[071d839](071d839)
and will continue to follow up on the refactors from PR #77206,
including a proper test to prevent regression.

---------

Co-authored-by: Zack Tanner <[email protected]>

Author: devjiwonchoi

packages/next/package.json

@@ -218,6 +218,7 @@
     "@types/ua-parser-js": "0.7.36",
     "@types/webpack-sources1": "npm:@types/[email protected]",
     "@types/ws": "8.2.0",
+    "@typescript/vfs": "1.6.1",
     "@vercel/ncc": "0.34.0",
     "@vercel/nft": "0.27.1",
     "@vercel/turbopack-ecmascript-runtime": "*",

packages/next/src/compiled/@typescript/vfs/LICENSE

@@ -0,0 +1,17 @@
+The MIT License (MIT)
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 
+associated documentation files (the "Software"), to deal in the Software without restriction, 
+including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial 
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT 
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/next/src/compiled/@typescript/vfs/index.js

@@ -0,0 +1 @@
+{"name":"@typescript/vfs","main":"index.js","author":"TypeScript team","license":"MIT"}

packages/next/src/compiled/@typescript/vfs/package.json

@@ -32,18 +32,6 @@ export const createTSPlugin: tsModule.server.PluginModuleFactory = ({
   typescript: ts,
 }) => {
   function create(info: tsModule.server.PluginCreateInfo) {
-    init({
-      ts,
-      info,
-    })
-
-    // Set up decorator object
-    const proxy = Object.create(null)
-    for (let k of Object.keys(info.languageService)) {
-      const x = (info.languageService as any)[k]
-      proxy[k] = (...args: Array<{}>) => x.apply(info.languageService, args)
-    }
-
     // Get plugin options
     // config is the plugin options from the user's tsconfig.json
     // e.g. { "plugins": [{ "name": "next", "enabled": true }] }
@@ -52,9 +40,22 @@ export const createTSPlugin: tsModule.server.PluginModuleFactory = ({
     const isPluginEnabled = info.config.enabled ?? true
 
     if (!isPluginEnabled) {
-      return proxy
+      return info.languageService
     }
 
+    // Set up decorator object
+    const proxy: tsModule.LanguageService = Object.create(null)
+    for (let k of Object.keys(info.languageService)) {
+      const x = info.languageService[k as keyof tsModule.LanguageService]
+      // @ts-expect-error - JS runtime trickery which is tricky to type tersely
+      proxy[k] = (...args: Array<{}>) => x.apply(info.languageService, args)
+    }
+
+    init({
+      ts,
+      info,
+    })
+
     // Auto completion
     proxy.getCompletionsAtPosition = (
       fileName: string,

packages/next/src/compiled/@typescript/vfs/typescript.js

@@ -1,10 +1,12 @@
 import { NEXT_TS_ERRORS } from '../constant'
 import {
-  getInfo,
   getSource,
+  getSourceFromVirtualTsEnv,
   getTs,
   getTypeChecker,
   isPositionInsideNode,
+  log,
+  virtualTsEnv,
 } from '../utils'
 
 import type tsModule from 'typescript/lib/tsserverlibrary'
@@ -49,101 +51,6 @@ function getMetadataExport(fileName: string, position: number) {
   return metadataExport
 }
 
-let cachedProxiedLanguageService: tsModule.LanguageService | undefined
-let cachedProxiedLanguageServiceHost: tsModule.LanguageServiceHost | undefined
-function getProxiedLanguageService() {
-  if (cachedProxiedLanguageService)
-    return {
-      languageService: cachedProxiedLanguageService as tsModule.LanguageService,
-      languageServiceHost:
-        cachedProxiedLanguageServiceHost as tsModule.LanguageServiceHost & {
-          addFile: (fileName: string, body: string) => void
-        },
-    }
-
-  const languageServiceHost = getInfo().languageServiceHost
-
-  const ts = getTs()
-  class ProxiedLanguageServiceHost implements tsModule.LanguageServiceHost {
-    files: {
-      [fileName: string]: { file: tsModule.IScriptSnapshot; ver: number }
-    } = {}
-
-    log = () => {}
-    trace = () => {}
-    error = () => {}
-    getCompilationSettings = () => languageServiceHost.getCompilationSettings()
-    getScriptIsOpen = () => true
-    getCurrentDirectory = () => languageServiceHost.getCurrentDirectory()
-    getDefaultLibFileName = (o: any) =>
-      languageServiceHost.getDefaultLibFileName(o)
-
-    getScriptVersion = (fileName: string) => {
-      const file = this.files[fileName]
-      if (!file) return languageServiceHost.getScriptVersion(fileName)
-      return file.ver.toString()
-    }
-
-    getScriptSnapshot = (fileName: string) => {
-      const file = this.files[fileName]
-      if (!file) return languageServiceHost.getScriptSnapshot(fileName)
-      return file.file
-    }
-
-    getScriptFileNames(): string[] {
-      const names: Set<string> = new Set()
-      for (var name in this.files) {
-        if (this.files.hasOwnProperty(name)) {
-          names.add(name)
-        }
-      }
-      const files = languageServiceHost.getScriptFileNames()
-      for (const file of files) {
-        names.add(file)
-      }
-      return [...names]
-    }
-
-    addFile(fileName: string, body: string) {
-      const snap = ts.ScriptSnapshot.fromString(body)
-      snap.getChangeRange = (_) => undefined
-      const existing = this.files[fileName]
-      if (existing) {
-        this.files[fileName].ver++
-        this.files[fileName].file = snap
-      } else {
-        this.files[fileName] = { ver: 1, file: snap }
-      }
-    }
-
-    readFile(fileName: string) {
-      const file = this.files[fileName]
-      return file
-        ? file.file.getText(0, file.file.getLength())
-        : languageServiceHost.readFile(fileName)
-    }
-    fileExists(fileName: string) {
-      return (
-        this.files[fileName] !== undefined ||
-        languageServiceHost.fileExists(fileName)
-      )
-    }
-  }
-
-  cachedProxiedLanguageServiceHost = new ProxiedLanguageServiceHost()
-  cachedProxiedLanguageService = ts.createLanguageService(
-    cachedProxiedLanguageServiceHost,
-    ts.createDocumentRegistry()
-  )
-  return {
-    languageService: cachedProxiedLanguageService as tsModule.LanguageService,
-    languageServiceHost:
-      cachedProxiedLanguageServiceHost as tsModule.LanguageServiceHost & {
-        addFile: (fileName: string, body: string) => void
-      },
-  }
-}
-
 function updateVirtualFileWithType(
   fileName: string,
   node: tsModule.VariableDeclaration | tsModule.FunctionDeclaration,
@@ -178,8 +85,17 @@ function updateVirtualFileWithType(
     annotation +
     sourceText.slice(nodeEnd) +
     TYPE_IMPORT
-  const { languageServiceHost } = getProxiedLanguageService()
-  languageServiceHost.addFile(fileName, newSource)
+
+  if (virtualTsEnv.sys.fileExists(fileName)) {
+    log('Updating file: ' + fileName)
+    // FIXME: updateFile() breaks as the file doesn't exists, which is weird.
+    // virtualTsEnv.updateFile(fileName, newSource)
+    virtualTsEnv.deleteFile(fileName)
+    virtualTsEnv.createFile(fileName, newSource)
+  } else {
+    log('Creating file: ' + fileName)
+    virtualTsEnv.createFile(fileName, newSource)
+  }
 
   return [nodeEnd, annotation.length]
 }
@@ -196,9 +112,9 @@ function proxyDiagnostics(
   n: tsModule.VariableDeclaration | tsModule.FunctionDeclaration
 ) {
   // Get diagnostics
-  const { languageService } = getProxiedLanguageService()
-  const diagnostics = languageService.getSemanticDiagnostics(fileName)
-  const source = getSource(fileName)
+  const diagnostics =
+    virtualTsEnv.languageService.getSemanticDiagnostics(fileName)
+  const source = getSourceFromVirtualTsEnv(fileName)
 
   // Filter and map the results
   return diagnostics
@@ -232,24 +148,26 @@ const metadata = {
     if (!node) return prior
     if (isTyped(node)) return prior
 
-    const ts = getTs()
-
     // We annotate with the type in a virtual language service
     const pos = updateVirtualFileWithType(fileName, node)
     if (pos === undefined) return prior
 
     // Get completions
-    const { languageService } = getProxiedLanguageService()
     const newPos = position <= pos[0] ? position : position + pos[1]
-    const completions = languageService.getCompletionsAtPosition(
+    const completions = virtualTsEnv.languageService.getCompletionsAtPosition(
       fileName,
       newPos,
       undefined
     )
 
     if (completions) {
+      const ts = getTs()
       completions.isIncomplete = true
-
+      // https://github.com/microsoft/TypeScript/blob/4dc677b292354f4b9162452b2e00f4d7dd118221/src/services/types.ts#L1428-L1433
+      if (completions.optionalReplacementSpan) {
+        // Adjust the start position of the text span to original source.
+        completions.optionalReplacementSpan.start -= newPos - position
+      }
       completions.entries = completions.entries
         .filter((e) => {
           return [
@@ -465,10 +383,9 @@ const metadata = {
     const pos = updateVirtualFileWithType(fileName, node)
     if (pos === undefined) return
 
-    const { languageService } = getProxiedLanguageService()
     const newPos = position <= pos[0] ? position : position + pos[1]
 
-    const details = languageService.getCompletionEntryDetails(
+    const details = virtualTsEnv.languageService.getCompletionEntryDetails(
       fileName,
       newPos,
       entryName,
@@ -489,9 +406,11 @@ const metadata = {
     const pos = updateVirtualFileWithType(fileName, node)
     if (pos === undefined) return
 
-    const { languageService } = getProxiedLanguageService()
     const newPos = position <= pos[0] ? position : position + pos[1]
-    const insight = languageService.getQuickInfoAtPosition(fileName, newPos)
+    const insight = virtualTsEnv.languageService.getQuickInfoAtPosition(
+      fileName,
+      newPos
+    )
     return insight
   },
 
@@ -503,11 +422,10 @@ const metadata = {
     // We annotate with the type in a virtual language service
     const pos = updateVirtualFileWithType(fileName, node)
     if (pos === undefined) return
-    const { languageService } = getProxiedLanguageService()
     const newPos = position <= pos[0] ? position : position + pos[1]
 
     const definitionInfoAndBoundSpan =
-      languageService.getDefinitionAndBoundSpan(fileName, newPos)
+      virtualTsEnv.languageService.getDefinitionAndBoundSpan(fileName, newPos)
 
     if (definitionInfoAndBoundSpan) {
       // Adjust the start position of the text span

packages/next/src/server/typescript/index.ts

@@ -1,28 +1,61 @@
-import path from 'path'
+import type { VirtualTypeScriptEnvironment } from 'next/dist/compiled/@typescript/vfs'
+import {
+  createFSBackedSystem,
+  createDefaultMapFromNodeModules,
+  createVirtualTypeScriptEnvironment,
+} from 'next/dist/compiled/@typescript/vfs'
+
+import path, { join } from 'path'
 
 import type tsModule from 'typescript/lib/tsserverlibrary'
 type TypeScript = typeof import('typescript/lib/tsserverlibrary')
 
 let ts: TypeScript
 let info: tsModule.server.PluginCreateInfo
 let appDirRegExp: RegExp
+export let virtualTsEnv: VirtualTypeScriptEnvironment
 
 export function log(message: string) {
-  info.project.projectService.logger.info(message)
+  info.project.projectService.logger.info('[next] ' + message)
 }
 
 // This function has to be called initially.
 export function init(opts: {
   ts: TypeScript
   info: tsModule.server.PluginCreateInfo
 }) {
+  const projectDir = opts.info.project.getCurrentDirectory()
   ts = opts.ts
   info = opts.info
-  const projectDir = info.project.getCurrentDirectory()
   appDirRegExp = new RegExp(
     '^' + (projectDir + '(/src)?/app').replace(/[\\/]/g, '[\\/]')
   )
-  log('Starting Next.js TypeScript plugin: ' + projectDir)
+
+  log('Initializing Next.js TypeScript plugin: ' + projectDir)
+
+  const compilerOptions = info.project.getCompilerOptions()
+  const fsMap = createDefaultMapFromNodeModules(
+    compilerOptions,
+    ts,
+    join(projectDir, 'node_modules/typescript/lib')
+  )
+  const system = createFSBackedSystem(fsMap, projectDir, ts)
+
+  virtualTsEnv = createVirtualTypeScriptEnvironment(
+    system,
+    [],
+    ts,
+    compilerOptions
+  )
+
+  if (virtualTsEnv) {
+    log(
+      'Failed to create virtual TypeScript environment. This is a bug in Next.js TypeScript plugin. Please report it by opening an issue at https://github.com/vercel/next.js/issues.'
+    )
+    return
+  }
+
+  log('Successfully initialized Next.js TypeScript plugin!')
 }
 
 export function getTs() {
@@ -41,6 +74,13 @@ export function getSource(fileName: string) {
   return info.languageService.getProgram()?.getSourceFile(fileName)
 }
 
+export function getSourceFromVirtualTsEnv(fileName: string) {
+  if (virtualTsEnv.sys.fileExists(fileName)) {
+    return virtualTsEnv.getSourceFile(fileName)
+  }
+  return getSource(fileName)
+}
+
 export function removeStringQuotes(str: string): string {
   return str.replace(/^['"`]|['"`]$/g, '')
 }

packages/next/src/server/typescript/rules/metadata.ts

@@ -2234,6 +2234,14 @@ export async function ncc_https_proxy_agent(task, opts) {
     .target('src/compiled/https-proxy-agent')
 }
 
+externals['@typescript/vfs'] = 'next/dist/compiled/@typescript/vfs'
+export async function ncc_typescript_vfs(task, opts) {
+  await task
+    .source(relative(__dirname, require.resolve('@typescript/vfs')))
+    .ncc({ packageName: '@typescript/vfs', externals })
+    .target('src/compiled/@typescript/vfs')
+}
+
 export async function precompile(task, opts) {
   await task.parallel(
     ['browser_polyfills', 'copy_ncced', 'copy_styled_jsx_assets'],
@@ -2381,6 +2389,7 @@ export async function ncc(task, opts) {
         'ncc_opentelemetry_api',
         'ncc_http_proxy_agent',
         'ncc_https_proxy_agent',
+        'ncc_typescript_vfs',
         'ncc_mini_css_extract_plugin',
       ],
       opts