[Turbopack] Compute module batches and use them for chunking (#76133)

### What?

Compute module batches and refactor chunking to work on the batches graph instead of the modules graph

With many pages we run into the problem that each page has a lot of modules, even if the total amount of modules is not super high. This multiples the number of edges needed in our graph.

To reduce that, group modules to module batches and module batches to batch groups and query information on these batches resp. batch groups to reduce the work needed.

A module batch will be formed from modules that are always used in a specific execution order.
A batch group will be formed from module batches that are in the same chunk groups.

Usually we work with module batches in a certain order and a unordered list of batch groups. We query information from the batch group and sort the results based on the order of module batches.

Author: sokra

Cargo.lock

@@ -1902,10 +1902,23 @@ impl Endpoint for AppEndpoint {
 
     #[turbo_tasks::function]
     async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
-        Ok(Vc::cell(vec![(
-            vec![self.app_endpoint_entry().await?.rsc_entry],
-            ChunkGroupType::Entry,
-        )]))
+        let this = self.await?;
+        Ok(Vc::cell(vec![
+            (
+                vec![self.app_endpoint_entry().await?.rsc_entry],
+                ChunkGroupType::Entry,
+            ),
+            (
+                this.app_project
+                    .client_runtime_entries()
+                    .await?
+                    .iter()
+                    .copied()
+                    .map(ResolvedVc::upcast)
+                    .collect(),
+                ChunkGroupType::Entry,
+            ),
+        ]))
     }
 
     #[turbo_tasks::function]

crates/next-api/src/app.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use serde::Serialize;
 use turbo_rcstr::RcStr;
 use turbo_tasks::{FxIndexMap, FxIndexSet, ResolvedVc, Vc};
-use turbopack_browser::ecmascript::EcmascriptDevChunk;
+use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
 use turbopack_core::{
     chunk::{Chunk, ChunkItem},
     output::OutputAsset,
@@ -27,7 +27,7 @@ where
             continue;
         };
 
-        if let Some(chunk) = ResolvedVc::try_downcast_type::<EcmascriptDevChunk>(*asset) {
+        if let Some(chunk) = ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(*asset) {
             let chunk_ident = normalize_client_path(&chunk.path().await?.path);
             chunks.push(WebpackStatsChunk {
                 size: asset_len,

crates/next-api/src/webpack_stats.rs

@@ -13,6 +13,7 @@ name = "mod"
 harness = false
 
 [features]
+default = []
 # Enables dynamic linking (and hot reloading) of embedded files/dirs.
 # A binary built with this option **is not portable**, the directory
 # path will be embedded into the binary.

turbopack/crates/turbo-tasks-fs/Cargo.toml

@@ -42,6 +42,7 @@ serde = { workspace = true, features = ["rc", "derive"] }
 serde_json = { workspace = true }
 serde_regex = "1.1.0"
 shrink-to-fit = { workspace=true,features = ["indexmap", "serde_json", "smallvec", "nightly"] }
+smallvec = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-util = { workspace = true }

turbopack/crates/turbo-tasks/Cargo.toml

@@ -1,6 +1,7 @@
 use std::fmt::{Debug, Display};
 
 use auto_hash_map::{AutoMap, AutoSet};
+use smallvec::SmallVec;
 use turbo_rcstr::RcStr;
 use turbo_tasks::{FxIndexMap, FxIndexSet, Vc};
 pub use turbo_tasks_macros::ValueDebugFormat;
@@ -157,6 +158,37 @@ where
     }
 }
 
+impl<T, const N: usize> ValueDebugFormat for SmallVec<[T; N]>
+where
+    T: ValueDebugFormat,
+{
+    fn value_debug_format(&self, depth: usize) -> ValueDebugFormatString {
+        if depth == 0 {
+            return ValueDebugFormatString::Sync(std::any::type_name::<Self>().to_string());
+        }
+
+        let values = self
+            .iter()
+            .map(|value| value.value_debug_format(depth.saturating_sub(1)))
+            .collect::<Vec<_>>();
+
+        ValueDebugFormatString::Async(Box::pin(async move {
+            let mut values_string = vec![];
+            for value in values {
+                match value {
+                    ValueDebugFormatString::Sync(string) => {
+                        values_string.push(PassthroughDebug::new_string(string));
+                    }
+                    ValueDebugFormatString::Async(future) => {
+                        values_string.push(PassthroughDebug::new_string(future.await?));
+                    }
+                }
+            }
+            Ok(format!("{:#?}", values_string))
+        }))
+    }
+}
+
 impl<K> ValueDebugFormat for AutoSet<K>
 where
     K: ValueDebugFormat,

turbopack/crates/turbo-tasks/src/debug/mod.rs

@@ -47,6 +47,7 @@ macro_rules! impl_auto_marker_trait {
 
         unsafe impl<T: $trait> $trait for ::std::option::Option<T> {}
         unsafe impl<T: $trait> $trait for ::std::vec::Vec<T> {}
+        unsafe impl<T: $trait, const N: usize> $trait for ::smallvec::SmallVec<[T; N]> {}
         unsafe impl<T: $trait, const N: usize> $trait for [T; N] {}
         unsafe impl<T: $trait> $trait for [T] {}
         unsafe impl<T: $trait, S> $trait for ::std::collections::HashSet<T, S> {}

turbopack/crates/turbo-tasks/src/marker_trait.rs

@@ -11,6 +11,7 @@ use std::{
 use auto_hash_map::{AutoMap, AutoSet};
 use either::Either;
 use indexmap::{IndexMap, IndexSet};
+use smallvec::SmallVec;
 use turbo_rcstr::RcStr;
 
 use crate::RawVc;
@@ -130,6 +131,14 @@ impl<T: TraceRawVcs> TraceRawVcs for Box<[T]> {
     }
 }
 
+impl<T: TraceRawVcs, const N: usize> TraceRawVcs for SmallVec<[T; N]> {
+    fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
+        for item in self.iter() {
+            TraceRawVcs::trace_raw_vcs(item, trace_context);
+        }
+    }
+}
+
 impl<T: TraceRawVcs, const N: usize> TraceRawVcs for [T; N] {
     fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
         for item in self.iter() {

turbopack/crates/turbo-tasks/src/trace.rs

@@ -4,6 +4,7 @@ use std::{
     future::IntoFuture,
     hash::{Hash, Hasher},
     marker::PhantomData,
+    mem::transmute,
     ops::Deref,
 };
 
@@ -205,6 +206,20 @@ where
             node: Vc::upcast(this.node),
         }
     }
+
+    /// Cheaply converts a Vec of resolved Vcs to a Vec of Vcs.
+    pub fn deref_vec(vec: Vec<ResolvedVc<T>>) -> Vec<Vc<T>> {
+        debug_assert!(size_of::<ResolvedVc<T>>() == size_of::<Vc<T>>());
+        // Safety: The memory layout of `ResolvedVc<T>` and `Vc<T>` is the same.
+        unsafe { transmute::<Vec<ResolvedVc<T>>, Vec<Vc<T>>>(vec) }
+    }
+
+    /// Cheaply converts a slice of resolved Vcs to a slice of Vcs.
+    pub fn deref_slice(slice: &[ResolvedVc<T>]) -> &[Vc<T>] {
+        debug_assert!(size_of::<ResolvedVc<T>>() == size_of::<Vc<T>>());
+        // Safety: The memory layout of `ResolvedVc<T>` and `Vc<T>` is the same.
+        unsafe { transmute::<&[ResolvedVc<T>], &[Vc<T>]>(slice) }
+    }
 }
 
 impl<T> ResolvedVc<T>

turbopack/crates/turbo-tasks/src/vc/resolved.rs

@@ -20,6 +20,7 @@ workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
+either = { workspace = true }
 indexmap = { workspace = true }
 indoc = { workspace = true }
 rustc-hash = { workspace = true }

turbopack/crates/turbopack-browser/Cargo.toml

@@ -27,8 +27,8 @@ use turbopack_ecmascript::{
 use turbopack_ecmascript_runtime::RuntimeType;
 
 use crate::ecmascript::{
-    chunk::EcmascriptDevChunk,
-    evaluate::chunk::EcmascriptDevEvaluateChunk,
+    chunk::EcmascriptBrowserChunk,
+    evaluate::chunk::EcmascriptBrowserEvaluateChunk,
     list::asset::{EcmascriptDevChunkList, EcmascriptDevChunkListSource},
 };
 
@@ -236,7 +236,7 @@ impl BrowserChunkingContext {
         // TODO(sokra) remove this argument and pass chunk items instead
         module_graph: Vc<ModuleGraph>,
     ) -> Vc<Box<dyn OutputAsset>> {
-        Vc::upcast(EcmascriptDevEvaluateChunk::new(
+        Vc::upcast(EcmascriptBrowserEvaluateChunk::new(
             self,
             ident,
             other_chunks,
@@ -271,7 +271,7 @@ impl BrowserChunkingContext {
             if let Some(ecmascript_chunk) =
                 Vc::try_resolve_downcast_type::<EcmascriptChunk>(chunk).await?
             {
-                Vc::upcast(EcmascriptDevChunk::new(self, ecmascript_chunk))
+                Vc::upcast(EcmascriptBrowserChunk::new(self, ecmascript_chunk))
             } else if let Some(output_asset) =
                 Vc::try_resolve_sidecast::<Box<dyn OutputAsset>>(chunk).await?
             {

turbopack/crates/turbopack-browser/src/chunking_context.rs

@@ -12,24 +12,24 @@ use turbopack_core::{
 };
 use turbopack_ecmascript::chunk::EcmascriptChunk;
 
-use crate::{ecmascript::content::EcmascriptDevChunkContent, BrowserChunkingContext};
+use crate::{ecmascript::content::EcmascriptBrowserChunkContent, BrowserChunkingContext};
 
 /// Development Ecmascript chunk.
 #[turbo_tasks::value(shared)]
-pub struct EcmascriptDevChunk {
+pub struct EcmascriptBrowserChunk {
     chunking_context: ResolvedVc<BrowserChunkingContext>,
     chunk: ResolvedVc<EcmascriptChunk>,
 }
 
 #[turbo_tasks::value_impl]
-impl EcmascriptDevChunk {
+impl EcmascriptBrowserChunk {
     /// Creates a new [`Vc<EcmascriptDevChunk>`].
     #[turbo_tasks::function]
     pub fn new(
         chunking_context: ResolvedVc<BrowserChunkingContext>,
         chunk: ResolvedVc<EcmascriptChunk>,
     ) -> Vc<Self> {
-        EcmascriptDevChunk {
+        EcmascriptBrowserChunk {
             chunking_context,
             chunk,
         }
@@ -38,15 +38,15 @@ impl EcmascriptDevChunk {
 }
 
 #[turbo_tasks::value_impl]
-impl ValueToString for EcmascriptDevChunk {
+impl ValueToString for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     fn to_string(&self) -> Vc<RcStr> {
         Vc::cell("Ecmascript Dev Chunk".into())
     }
 }
 
 #[turbo_tasks::value_impl]
-impl OutputChunk for EcmascriptDevChunk {
+impl OutputChunk for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     async fn runtime_info(&self) -> Result<Vc<OutputChunkRuntimeInfo>> {
         Ok(OutputChunkRuntimeInfo {
@@ -63,11 +63,11 @@ fn modifier() -> Vc<RcStr> {
 }
 
 #[turbo_tasks::value_impl]
-impl EcmascriptDevChunk {
+impl EcmascriptBrowserChunk {
     #[turbo_tasks::function]
-    async fn own_content(self: Vc<Self>) -> Result<Vc<EcmascriptDevChunkContent>> {
+    async fn own_content(self: Vc<Self>) -> Result<Vc<EcmascriptBrowserChunkContent>> {
         let this = self.await?;
-        Ok(EcmascriptDevChunkContent::new(
+        Ok(EcmascriptBrowserChunkContent::new(
             *this.chunking_context,
             self,
             this.chunk.chunk_content(),
@@ -81,7 +81,7 @@ impl EcmascriptDevChunk {
 }
 
 #[turbo_tasks::value_impl]
-impl OutputAsset for EcmascriptDevChunk {
+impl OutputAsset for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     fn path(&self) -> Vc<FileSystemPath> {
         let ident = self.chunk.ident().with_modifier(modifier());
@@ -117,7 +117,7 @@ impl OutputAsset for EcmascriptDevChunk {
 }
 
 #[turbo_tasks::value_impl]
-impl Asset for EcmascriptDevChunk {
+impl Asset for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     fn content(self: Vc<Self>) -> Vc<AssetContent> {
         self.own_content().content()
@@ -130,7 +130,7 @@ impl Asset for EcmascriptDevChunk {
 }
 
 #[turbo_tasks::value_impl]
-impl GenerateSourceMap for EcmascriptDevChunk {
+impl GenerateSourceMap for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     fn generate_source_map(self: Vc<Self>) -> Vc<OptionStringifiedSourceMap> {
         self.own_content().generate_source_map()
@@ -153,7 +153,7 @@ fn introspectable_details() -> Vc<RcStr> {
 }
 
 #[turbo_tasks::value_impl]
-impl Introspectable for EcmascriptDevChunk {
+impl Introspectable for EcmascriptBrowserChunk {
     #[turbo_tasks::function]
     fn ty(&self) -> Vc<RcStr> {
         introspectable_type()