Turbopack: Support data URI sources (#76865)

These are technically not entirely spec compliant as identical data urls in different folders aren't deduplicated (but each generate a new module). But this is good enough for now.

You can now do 
```js
import { bar } from "data:text/javascript,export var bar = 1234;";
import data from 'data:application/json,{ "foo": 1234 }';
import dataURLEncoded from 'data:application/json,%7B%20%22foo%22%3A%201234%20%7D';
import dataBase64Encoded from 'data:application/json;base64,eyAiZm9vIjogMTIzNCB9';
import "data:text/css,body { color: red }";

it("support data URL imports", () => {
  expect(bar).toEqual(1234);
  expect(data).toEqual({ foo: 1234 });
  expect(dataURLEncoded).toEqual({ foo: 1234 });
  expect(dataBase64Encoded).toEqual({ foo: 1234 });
});
```

Only data-urls in JS ESM imports are turned into modules. For JS, turning data-urls into separate modules is zero-cost, because they all end up in the same bundle anyway, but that is not the case for image data urls in CSS.

Author: mischnic

Diff for: Cargo.lock

@@ -342,6 +342,7 @@ const_format = "0.2.30"
 criterion = "0.5.1"
 crossbeam-channel = "0.5.8"
 dashmap = "6.1.0"
+data-encoding = "2.3.3"
 dhat = { version = "0.3.2" }
 dialoguer = "0.10.3"
 dunce = "1.0.3"

Diff for: Cargo.toml

@@ -18,6 +18,7 @@ async-trait = { workspace = true }
 auto-hash-map = { workspace = true }
 browserslist-rs = { workspace = true }
 const_format = { workspace = true }
+data-encoding = { workspace = true }
 either = { workspace = true }
 futures = { workspace = true }
 indexmap = { workspace = true }
@@ -42,6 +43,7 @@ turbo-tasks = { workspace = true }
 turbo-tasks-env = { workspace = true }
 turbo-tasks-fs = { workspace = true }
 turbo-tasks-hash = { workspace = true }
+urlencoding = { workspace = true }
 
 [build-dependencies]
 turbo-tasks-build = { workspace = true }

Diff for: turbopack/crates/turbopack-core/Cargo.toml

@@ -0,0 +1,80 @@
+use anyhow::{bail, Result};
+use turbo_rcstr::RcStr;
+use turbo_tasks::{ResolvedVc, Vc};
+use turbo_tasks_fs::{rope::Rope, File, FileContent, FileSystemPath};
+use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
+
+use crate::{
+    asset::{Asset, AssetContent},
+    ident::AssetIdent,
+    source::Source,
+};
+
+/// The raw [Source]. It represents raw content from a path without any
+/// references to other [Source]s.
+#[turbo_tasks::value]
+pub struct DataUriSource {
+    media_type: RcStr,
+    encoding: RcStr,
+    data: ResolvedVc<RcStr>,
+    lookup_path: ResolvedVc<FileSystemPath>,
+}
+
+#[turbo_tasks::value_impl]
+impl DataUriSource {
+    #[turbo_tasks::function]
+    pub fn new(
+        media_type: RcStr,
+        encoding: RcStr,
+        data: ResolvedVc<RcStr>,
+        lookup_path: ResolvedVc<FileSystemPath>,
+    ) -> Vc<Self> {
+        Self::cell(DataUriSource {
+            media_type,
+            encoding,
+            data,
+            lookup_path,
+        })
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl Source for DataUriSource {
+    #[turbo_tasks::function]
+    async fn ident(&self) -> Result<Vc<AssetIdent>> {
+        let content_type = self.media_type.split(";").next().unwrap().into();
+        let filename = format!(
+            "data:{}",
+            &encode_hex(hash_xxh3_hash64((
+                &*self.data.await?,
+                &self.media_type,
+                &self.encoding
+            )))[0..6]
+        );
+        Ok(
+            AssetIdent::from_path(self.lookup_path.join(filename.into()))
+                .with_content_type(content_type),
+        )
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl Asset for DataUriSource {
+    #[turbo_tasks::function]
+    async fn content(&self) -> Result<Vc<AssetContent>> {
+        let data = self.data.await?;
+        let rope = if self.encoding == "base64" {
+            let decoded = data_encoding::BASE64.decode(data.as_bytes())?;
+            // TODO this should read self.media_type and potentially use a different encoding
+            Rope::from(decoded)
+        } else if self.encoding.is_empty() {
+            let decoded = urlencoding::decode(data.as_str())?.into_owned();
+            Rope::from(decoded)
+        } else {
+            bail!("Unsupported data URL encoding: {}", self.encoding);
+        };
+        Ok(AssetContent::file(
+            FileContent::from(File::from(rope)).cell(),
+        ))
+    }
+}

Diff for: turbopack/crates/turbopack-core/src/data_uri_source.rs

@@ -27,6 +27,8 @@ pub struct AssetIdent {
     pub parts: Vec<ModulePart>,
     /// The asset layer the asset was created from.
     pub layer: Option<ResolvedVc<RcStr>>,
+    /// The MIME content type, if this asset was created from a data URL.
+    pub content_type: Option<RcStr>,
 }
 
 impl AssetIdent {
@@ -98,6 +100,10 @@ impl ValueToString for AssetIdent {
             s.push(')');
         }
 
+        if let Some(content_type) = &self.content_type {
+            write!(s, " <{}>", content_type)?;
+        }
+
         if !self.parts.is_empty() {
             for part in self.parts.iter() {
                 if !matches!(part, ModulePart::Facade) {
@@ -130,6 +136,7 @@ impl AssetIdent {
             modifiers: Vec::new(),
             parts: Vec::new(),
             layer: None,
+            content_type: None,
         }))
     }
 
@@ -168,6 +175,13 @@ impl AssetIdent {
         Self::new(Value::new(this))
     }
 
+    #[turbo_tasks::function]
+    pub fn with_content_type(&self, content_type: RcStr) -> Vc<Self> {
+        let mut this = self.clone();
+        this.content_type = Some(content_type);
+        Self::new(Value::new(this))
+    }
+
     #[turbo_tasks::function]
     pub async fn rename_as(&self, pattern: RcStr) -> Result<Vc<Self>> {
         let mut this = self.clone();
@@ -230,6 +244,7 @@ impl AssetIdent {
             modifiers,
             parts,
             layer,
+            content_type,
         } = self;
         let query = query.await?;
         if !query.is_empty() {
@@ -303,10 +318,15 @@ impl AssetIdent {
             has_hash = true;
         }
         if let Some(layer) = layer {
-            1_u8.deterministic_hash(&mut hasher);
+            5_u8.deterministic_hash(&mut hasher);
             layer.await?.deterministic_hash(&mut hasher);
             has_hash = true;
         }
+        if let Some(content_type) = content_type {
+            6_u8.deterministic_hash(&mut hasher);
+            content_type.deterministic_hash(&mut hasher);
+            has_hash = true;
+        }
 
         if has_hash {
             let hash = encode_hex(hasher.finish());

Diff for: turbopack/crates/turbopack-core/src/ident.rs

@@ -14,6 +14,7 @@ pub mod code_builder;
 pub mod compile_time_info;
 pub mod condition;
 pub mod context;
+pub mod data_uri_source;
 pub mod diagnostics;
 pub mod environment;
 pub mod error;

Diff for: turbopack/crates/turbopack-core/src/lib.rs

@@ -32,6 +32,7 @@ use self::{
 };
 use crate::{
     context::AssetContext,
+    data_uri_source::DataUriSource,
     file_source::FileSource,
     issue::{
         module::emit_unknown_module_type_error, resolve::ResolvingIssue, IssueExt, IssueSource,
@@ -43,6 +44,7 @@ use crate::{
     reference_type::ReferenceType,
     resolve::{
         node::{node_cjs_resolve_options, node_esm_resolve_options},
+        parse::stringify_data_uri,
         pattern::{read_matches, PatternMatch},
         plugin::AfterResolvePlugin,
     },
@@ -1998,6 +2000,40 @@ async fn resolve_internal_inline(
                 )
                 .await?
             }
+            Request::DataUri {
+                media_type,
+                encoding,
+                data,
+            } => {
+                // Behave like Request::Uri
+                let uri: RcStr = stringify_data_uri(media_type, encoding, *data)
+                    .await?
+                    .into();
+                if options.await?.parse_data_uris {
+                    *ResolveResult::primary_with_key(
+                        RequestKey::new(uri.clone()),
+                        ResolveResultItem::Source(ResolvedVc::upcast(
+                            DataUriSource::new(
+                                media_type.clone(),
+                                encoding.clone(),
+                                **data,
+                                lookup_path,
+                            )
+                            .to_resolved()
+                            .await?,
+                        )),
+                    )
+                } else {
+                    *ResolveResult::primary_with_key(
+                        RequestKey::new(uri.clone()),
+                        ResolveResultItem::External {
+                            name: uri,
+                            ty: ExternalType::Url,
+                            traced: ExternalTraced::Untraced,
+                        },
+                    )
+                }
+            }
             Request::Uri {
                 protocol,
                 remainder,

Diff for: turbopack/crates/turbopack-core/src/resolve/mod.rs

@@ -602,6 +602,8 @@ pub struct ResolveOptions {
     pub enable_typescript_with_output_extension: bool,
     /// Warn instead of error for resolve errors
     pub loose_errors: bool,
+    /// Whether to parse data URIs into modules (as opposed to keeping them as externals)
+    pub parse_data_uris: bool,
 
     pub placeholder_for_future_extensions: (),
 }