[devtools] Add a query parameter to restart endpoint to invalidate the persistent cache (#79425)

Creates and passes through an option to turbopack to invalidate the cache. The dev overlay can then (not implemented) call this endpoint.

Invalidation is performed by writing a marker file to disk, and then actually deleting the cache upon the next startup because:

- Getting the database to drop all file handles is tricky.
- You can't delete open files on Windows.
- Writing an invalidation file is atomic, recursively deleting a directory is not.

### Testing

Set up a small app with a large dependency (three.js). Ran the build, restarted the dev server, saw that the cached run was fast.

Ran

```
curl -v --request POST --header "Content-Type: application/json" --data '{}' http://localhost:3000/__nextjs_restart_dev\?invalidatePersistentCache\=
```

And saw that the server restarted and the next page load was slow.

### Remaining Work (will come in subsequent PRs)

In rough order of priority:

- A separate endpoint to poll to see when the server is back up.
- A webpack+rspack implementation. It looks like I can do a hack to forcibly change the webpack cache version, which should trigger similar behavior upon the next restart.
- Passing a boolean to the dev overlay telling it if persistent caching is enabled or not.
- The devtools UI implementation.
- Telemetry.
- A CLI subcommand that leverages this, so that you can do it from the terminal.

Author: bgw

crates/napi/src/next_api/project.rs

@@ -568,6 +568,18 @@ pub async fn project_update(
     Ok(())
 }
 
+/// Invalidates the persistent cache so that it will be deleted next time that a turbopack project
+/// is created with persistent caching enabled.
+#[napi]
+pub async fn project_invalidate_persistent_cache(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+) -> napi::Result<()> {
+    tokio::task::spawn_blocking(move || project.turbo_tasks.invalidate_persistent_cache())
+        .await
+        .context("panicked while invalidating persistent cache")??;
+    Ok(())
+}
+
 /// Runs exit handlers for the project registered using the [`ExitHandler`] API.
 ///
 /// This is called by `project_shutdown`, so if you're calling that API, you shouldn't call this

crates/napi/src/next_api/utils.rs

@@ -149,6 +149,16 @@ impl NextTurboTasks {
             }
         }
     }
+
+    pub fn invalidate_persistent_cache(&self) -> Result<()> {
+        match self {
+            NextTurboTasks::Memory(_) => {}
+            NextTurboTasks::PersistentCaching(turbo_tasks) => {
+                turbo_tasks.backend().invalidate_storage()?
+            }
+        }
+        Ok(())
+    }
 }
 
 pub fn create_turbo_tasks(

packages/next/src/build/swc/generated-native.d.ts

@@ -204,6 +204,13 @@ export declare function projectUpdate(
   project: { __napiType: 'Project' },
   options: NapiPartialProjectOptions
 ): Promise<void>
+/**
+ * Invalidates the persistent cache so that it will be deleted next time that a turbopack project
+ * is created with persistent caching enabled.
+ */
+export declare function projectInvalidatePersistentCache(project: {
+  __napiType: 'Project'
+}): Promise<void>
 /**
  * Runs exit handlers for the project registered using the [`ExitHandler`] API.
  *

packages/next/src/build/swc/index.ts

@@ -723,6 +723,10 @@ function bindingToApi(
       )
     }
 
+    invalidatePersistentCache(): Promise<void> {
+      return binding.projectInvalidatePersistentCache(this._nativeProject)
+    }
+
     shutdown(): Promise<void> {
       return binding.projectShutdown(this._nativeProject)
     }

packages/next/src/build/swc/types.ts

@@ -226,6 +226,8 @@ export interface Project {
     TurbopackResult<CompilationEvent>
   >
 
+  invalidatePersistentCache(): Promise<void>
+
   shutdown(): Promise<void>
 
   onExit(): Promise<void>

packages/next/src/client/components/react-dev-overlay/server/restart-dev-server-middleware.ts

@@ -2,23 +2,39 @@ import type { ServerResponse, IncomingMessage } from 'http'
 import type { Telemetry } from '../../../../telemetry/storage'
 import { RESTART_EXIT_CODE } from '../../../../server/lib/utils'
 import { middlewareResponse } from './middleware-response'
+import type { Project } from '../../../../build/swc/types'
 
 const EVENT_DEV_OVERLAY_RESTART_SERVER = 'DEV_OVERLAY_RESTART_SERVER'
 
-export function getRestartDevServerMiddleware(telemetry: Telemetry) {
+interface RestartDevServerMiddlewareConfig {
+  telemetry: Telemetry
+  turbopackProject?: Project
+}
+
+export function getRestartDevServerMiddleware({
+  telemetry,
+  turbopackProject,
+}: RestartDevServerMiddlewareConfig) {
   return async function (
     req: IncomingMessage,
     res: ServerResponse,
     next: () => void
   ): Promise<void> {
-    const { pathname } = new URL(`http://n${req.url}`)
+    const { pathname, searchParams } = new URL(`http://n${req.url}`)
     if (pathname !== '/__nextjs_restart_dev' || req.method !== 'POST') {
       return next()
     }
 
+    const invalidatePersistentCache = searchParams.has(
+      'invalidatePersistentCache'
+    )
+    if (invalidatePersistentCache) {
+      await turbopackProject?.invalidatePersistentCache()
+    }
+
     telemetry.record({
       eventName: EVENT_DEV_OVERLAY_RESTART_SERVER,
-      payload: {},
+      payload: { invalidatePersistentCache },
     })
 
     // TODO: Use flushDetached

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -650,7 +650,10 @@ export async function createHotReloaderTurbopack(
     getNextErrorFeedbackMiddleware(opts.telemetry),
     getDevOverlayFontMiddleware(),
     getDisableDevIndicatorMiddleware(),
-    getRestartDevServerMiddleware(opts.telemetry),
+    getRestartDevServerMiddleware({
+      telemetry: opts.telemetry,
+      turbopackProject: project,
+    }),
   ]
 
   const versionInfoPromise = getVersionInfo()

packages/next/src/server/dev/hot-reloader-webpack.ts

@@ -1570,7 +1570,9 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
       getNextErrorFeedbackMiddleware(this.telemetry),
       getDevOverlayFontMiddleware(),
       getDisableDevIndicatorMiddleware(),
-      getRestartDevServerMiddleware(this.telemetry),
+      getRestartDevServerMiddleware({
+        telemetry: this.telemetry,
+      }),
     ]
   }
 

turbopack/crates/turbo-persistence/src/db.rs

@@ -190,7 +190,7 @@ impl TurboPersistence {
         Ok(db)
     }
 
-    /// Performas the initial check on the database directory.
+    /// Performs the initial check on the database directory.
     fn open_directory(&mut self) -> Result<()> {
         match fs::read_dir(&self.path) {
             Ok(entries) => {

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -207,6 +207,10 @@ impl<B: BackingStorage> TurboTasksBackend<B> {
             backing_storage,
         )))
     }
+
+    pub fn invalidate_storage(&self) -> Result<()> {
+        self.0.backing_storage.invalidate()
+    }
 }
 
 impl<B: BackingStorage> TurboTasksBackendInner<B> {

turbopack/crates/turbo-tasks-backend/src/backing_storage.rs

@@ -63,6 +63,16 @@ pub trait BackingStorage: 'static + Send + Sync {
         category: TaskDataCategory,
     ) -> Result<Vec<CachedDataItem>>;
 
+    /// Called when the database should be invalidated upon re-initialization.
+    ///
+    /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
+    /// database. If this happens, there's no point in writing anything else to disk, or flushing
+    /// during [`KeyValueDatabase::shutdown`].
+    ///
+    /// This can be implemented by calling [`crate::database::db_invalidation::invalidate_db`] with
+    /// the database's non-versioned base path.
+    fn invalidate(&self) -> Result<()>;
+
     fn shutdown(&self) -> Result<()> {
         Ok(())
     }

turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs

@@ -0,0 +1,69 @@
+use std::{
+    fs::{self, read_dir},
+    io,
+    path::Path,
+};
+
+use anyhow::Context;
+
+const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
+
+/// Atomically write an invalidation marker.
+///
+/// Because attempting to delete currently open database files could cause issues, actual deletion
+/// of files is deferred until the next start-up (in [`check_db_invalidation_and_cleanup`]).
+///
+/// In the case that no database is currently open (e.g. via a separate CLI subcommand), you should
+/// call [`cleanup_db`] *after* this to eagerly remove the database files.
+///
+/// This should be run with the base (non-versioned) path, as that likely aligns closest with user
+/// expectations (e.g. if they're clearing the cache for disk space reasons).
+pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> {
+    fs::write(base_path.join(INVALIDATION_MARKER), [0u8; 0])?;
+    Ok(())
+}
+
+/// Called during startup. See if the db is in a partially-completed invalidation state. Find and
+/// delete any invalidated database files.
+///
+/// This should be run with the base (non-versioned) path.
+pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> anyhow::Result<()> {
+    if fs::exists(base_path.join(INVALIDATION_MARKER))? {
+        // if this cleanup fails, we might try to open an invalid database later, so it's best to
+        // just propagate the error here.
+        cleanup_db(base_path)?;
+    };
+    Ok(())
+}
+
+/// Helper for [`check_db_invalidation_and_cleanup`]. You can call this to explicitly clean up a
+/// database after running [`invalidate_db`] when turbo-tasks is not running.
+///
+/// You should not run this if the database has not yet been invalidated, as this operation is not
+/// atomic and could result in a partially-deleted and corrupted database.
+pub fn cleanup_db(base_path: &Path) -> anyhow::Result<()> {
+    cleanup_db_inner(base_path).with_context(|| {
+        format!(
+            "Unable to remove invalid database. If this issue persists you can work around by \
+             deleting {base_path:?}."
+        )
+    })
+}
+
+fn cleanup_db_inner(base_path: &Path) -> io::Result<()> {
+    let Ok(contents) = read_dir(base_path) else {
+        return Ok(());
+    };
+
+    // delete everything except the invalidation marker
+    for entry in contents {
+        let entry = entry?;
+        if entry.file_name() != INVALIDATION_MARKER {
+            fs::remove_dir_all(entry.path())?;
+        }
+    }
+
+    // delete the invalidation marker last, once we're sure everything is cleaned up
+    fs::remove_file(base_path.join(INVALIDATION_MARKER))?;
+    Ok(())
+}

turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs

@@ -52,6 +52,20 @@ pub trait KeyValueDatabase {
         &self,
     ) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>>;
 
+    /// Called when the database has been invalidated via
+    /// [`crate::backing_storage::BackingStorage::invalidate`]
+    ///
+    /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
+    /// database. If this happens, there's no point in writing anything else to disk, or flushing
+    /// during [`KeyValueDatabase::shutdown`].
+    ///
+    /// This is a best-effort optimization hint, and the database may choose to ignore this and
+    /// continue file writes. This happens after the database is invalidated, so it is valid for
+    /// this to leave the database in a half-updated and corrupted state.
+    fn prevent_writes(&self) {
+        // this is an optional performance hint to the database
+    }
+
     fn shutdown(&self) -> Result<()> {
         Ok(())
     }

turbopack/crates/turbo-tasks-backend/src/database/mod.rs

@@ -1,5 +1,6 @@
 #[cfg(feature = "lmdb")]
 mod by_key_space;
+pub mod db_invalidation;
 pub mod db_versioning;
 #[cfg(feature = "lmdb")]
 pub mod fresh_db_optimization;

turbopack/crates/turbo-tasks-backend/src/database/turbo.rs

@@ -23,8 +23,8 @@ pub struct TurboKeyValueDatabase {
 }
 
 impl TurboKeyValueDatabase {
-    pub fn new(path: PathBuf) -> Result<Self> {
-        let db = Arc::new(TurboPersistence::open(path.to_path_buf())?);
+    pub fn new(versioned_path: PathBuf) -> Result<Self> {
+        let db = Arc::new(TurboPersistence::open(versioned_path)?);
         let mut this = Self {
             db: db.clone(),
             compact_join_handle: Mutex::new(None),
@@ -99,6 +99,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         }))
     }
 
+    fn prevent_writes(&self) {}
+
     fn shutdown(&self) -> Result<()> {
         // Wait for the compaction to finish
         if let Some(join_handle) = self.compact_join_handle.lock().take() {

turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Borrow, cmp::max, sync::Arc};
+use std::{borrow::Borrow, cmp::max, path::PathBuf, sync::Arc};
 
 use anyhow::{Context, Result, anyhow};
 use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
@@ -8,10 +8,13 @@ use tracing::Span;
 use turbo_tasks::{SessionId, TaskId, backend::CachedTaskType, turbo_tasks_scope};
 
 use crate::{
+    GitVersionInfo,
     backend::{AnyOperation, TaskDataCategory},
     backing_storage::BackingStorage,
     data::CachedDataItem,
     database::{
+        db_invalidation::{check_db_invalidation_and_cleanup, invalidate_db},
+        db_versioning::handle_db_versioning,
         key_value_database::{KeySpace, KeyValueDatabase},
         write_batch::{
             BaseWriteBatch, ConcurrentWriteBatch, SerialWriteBatch, WriteBatch, WriteBatchRef,
@@ -82,11 +85,31 @@ fn as_u32(bytes: impl Borrow<[u8]>) -> Result<u32> {
 
 pub struct KeyValueDatabaseBackingStorage<T: KeyValueDatabase> {
     database: T,
+    /// Used when calling [`BackingStorage::invalidate`]. Can be `None` in the memory-only/no-op
+    /// storage case.
+    base_path: Option<PathBuf>,
 }
 
 impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
-    pub fn new(database: T) -> Self {
-        Self { database }
+    pub fn new_in_memory(database: T) -> Self {
+        Self {
+            database,
+            base_path: None,
+        }
+    }
+
+    pub fn open_versioned_on_disk(
+        base_path: PathBuf,
+        version_info: &GitVersionInfo,
+        is_ci: bool,
+        database: impl FnOnce(PathBuf) -> Result<T>,
+    ) -> Result<Self> {
+        check_db_invalidation_and_cleanup(&base_path)?;
+        let versioned_path = handle_db_versioning(&base_path, version_info, is_ci)?;
+        Ok(Self {
+            database: (database)(versioned_path)?,
+            base_path: Some(base_path),
+        })
     }
 
     fn with_tx<R>(
@@ -442,6 +465,18 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
             .with_context(|| format!("Looking up data for {task_id} from database failed"))
     }
 
+    fn invalidate(&self) -> Result<()> {
+        // `base_path` can be `None` for a `NoopKvDb`
+        if let Some(base_path) = &self.base_path {
+            // Invalidate first, as it's a very fast atomic operation. `prevent_writes` is allowed
+            // to be slower (e.g. wait for a lock) and is allowed to corrupt the database with
+            // partial writes.
+            invalidate_db(base_path)?;
+            self.database.prevent_writes()
+        }
+        Ok(())
+    }
+
     fn shutdown(&self) -> Result<()> {
         self.database.shutdown()
     }