WIP: Turbopack: Implement bincode Encode/Decode traits on all turbo task values

Author: bgw

Cargo.lock

@@ -359,6 +359,7 @@ preset_env_base = "5.0.0"
 
 
 # General Deps
+bincode = { version = "2.0.1", features = ["serde"] }
 chromiumoxide = { version = "0.5.4", features = [
   "tokio-runtime",
 ], default-features = false }
@@ -480,4 +481,6 @@ webbrowser = "1.0.6"
 inventory = "0.3.21"
 
 [patch.crates-io]
+bincode = { git = "https://github.com/bgw/bincode.git", branch = "bgw/patches" }
+virtue = { git = "https://github.com/bgw/virtue.git", branch = "bgw/fix-generic-default-parsing" }
 mdxjs = { git = "https://github.com/mischnic/mdxjs-rs.git", branch = "swc-core-32" }

Cargo.toml

@@ -10,6 +10,7 @@ atom_size_128 = []
 napi = ["dep:napi"]
 
 [dependencies]
+bincode = { workspace = true }
 triomphe = { workspace = true }
 turbo-tasks-hash = { workspace = true }
 serde = { workspace = true }

turbopack/crates/turbo-rcstr/Cargo.toml

@@ -9,6 +9,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
+use bincode::{Decode, Encode, de::Decoder, enc::Encoder, error::EncodeError, impl_borrow_decode};
 use bytes_str::BytesStr;
 use debug_unreachable::debug_unreachable;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -369,6 +370,22 @@ impl<'de> Deserialize<'de> for RcStr {
     }
 }
 
+impl Encode for RcStr {
+    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        self.as_str().encode(encoder)
+    }
+}
+
+impl<Context> Decode<Context> for RcStr {
+    fn decode<D: Decoder<Context = Context>>(
+        decoder: &mut D,
+    ) -> Result<Self, bincode::error::DecodeError> {
+        Ok(RcStr::from(String::decode(decoder)?))
+    }
+}
+
+impl_borrow_decode!(RcStr);
+
 impl Drop for RcStr {
     fn drop(&mut self) {
         match self.tag() {

turbopack/crates/turbo-rcstr/src/lib.rs

@@ -9,6 +9,7 @@ edition = "2024"
 workspace = true
 
 [dependencies]
+bincode = { workspace = true }
 hashbrown = { workspace = true, features = ["serde"]}
 rustc-hash = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

turbopack/crates/turbo-tasks-auto-hash-map/Cargo.toml

@@ -5,6 +5,13 @@ use std::{
     marker::PhantomData,
 };
 
+use bincode::{
+    Decode, Encode,
+    de::Decoder,
+    enc::Encoder,
+    error::{DecodeError, EncodeError},
+    impl_borrow_decode,
+};
 use hashbrown::hash_map::HashMap;
 use rustc_hash::FxHasher;
 use serde::{
@@ -54,7 +61,7 @@ impl<K, V> AutoMap<K, V, BuildHasherDefault<FxHasher>, 0> {
     }
 }
 
-impl<K, V, H: BuildHasher, const I: usize> AutoMap<K, V, H, I> {
+impl<K, V, H, const I: usize> AutoMap<K, V, H, I> {
     /// see [HashMap::with_hasher](https://doc.rust-lang.org/std/collections/hash_map/struct.HashMap.html#method.with_hasher)
     pub const fn with_hasher() -> Self {
         AutoMap::List(SmallVec::new_const())
@@ -813,6 +820,56 @@ where
     }
 }
 
+impl<K, V, H, const I: usize> Encode for AutoMap<K, V, H, I>
+where
+    K: Encode,
+    V: Encode,
+{
+    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        // Encode the length first
+        self.len().encode(encoder)?;
+        // Then encode each key-value tuple
+        for entry in self.iter() {
+            entry.encode(encoder)?;
+        }
+        Ok(())
+    }
+}
+
+impl<Context, K, V, H, const I: usize> Decode<Context> for AutoMap<K, V, H, I>
+where
+    K: Decode<Context> + Eq + Hash,
+    V: Decode<Context>,
+    H: BuildHasher + Default,
+{
+    fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
+        let len = usize::decode(decoder)?;
+        if len <= MAX_LIST_SIZE {
+            let mut list = SmallVec::with_capacity(len);
+            for _ in 0..len {
+                let entry = <(K, V)>::decode(decoder)?;
+                list.push(entry);
+            }
+            Ok(AutoMap::List(list))
+        } else {
+            let mut map = HashMap::with_capacity_and_hasher(len, H::default());
+            for _ in 0..len {
+                let (key, value) = <(K, V)>::decode(decoder)?;
+                map.insert(key, value);
+            }
+            Ok(AutoMap::Map(Box::new(map)))
+        }
+    }
+}
+
+impl_borrow_decode!(
+    AutoMap<K, V, H, I>,
+    K: Decode<__Context> + Eq + Hash,
+    V: Decode<__Context>,
+    H: BuildHasher + Default,
+    const I: usize,
+);
+
 impl<K: Eq + Hash, V: Eq, H: BuildHasher, const I: usize> PartialEq for AutoMap<K, V, H, I> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {

turbopack/crates/turbo-tasks-auto-hash-map/src/map.rs

@@ -4,13 +4,19 @@ use std::{
     marker::PhantomData,
 };
 
+use bincode::{Decode, Encode};
 use rustc_hash::FxHasher;
 use serde::{Deserialize, Serialize};
 use shrink_to_fit::ShrinkToFit;
 
 use crate::AutoMap;
 
-#[derive(Clone)]
+#[derive(Clone, Encode, Decode)]
+#[bincode(
+    encode_bounds = "K: Encode + Hash + Eq, H: BuildHasher + Default",
+    decode_bounds = "K: Decode<__Context> + Hash + Eq, H: BuildHasher + Default",
+    borrow_decode_bounds = "K: Decode<__Context> + Hash + Eq, H: BuildHasher + Default"
+)]
 pub struct AutoSet<K, H = BuildHasherDefault<FxHasher>, const I: usize = 0> {
     map: AutoMap<K, (), H, I>,
 }

turbopack/crates/turbo-tasks-auto-hash-map/src/set.rs

@@ -18,7 +18,11 @@ pub use self::{
 };
 
 #[turbo_tasks::value(transparent)]
-pub struct EnvMap(#[turbo_tasks(trace_ignore)] FxIndexMap<RcStr, RcStr>);
+pub struct EnvMap(
+    #[turbo_tasks(trace_ignore)]
+    #[bincode(with = "turbo_tasks::bincode::indexmap")]
+    FxIndexMap<RcStr, RcStr>,
+);
 
 #[turbo_tasks::value_impl]
 impl EnvMap {

turbopack/crates/turbo-tasks-env/src/lib.rs

@@ -25,6 +25,7 @@ workspace = true
 [dependencies]
 anyhow = { workspace = true }
 auto-hash-map = { workspace = true }
+bincode = { workspace = true }
 bitflags = "1.3.2"
 bytes = { workspace = true }
 concurrent-queue = { workspace = true }

turbopack/crates/turbo-tasks-fs/Cargo.toml

@@ -1,6 +1,12 @@
 use std::fmt::Display;
 
 use anyhow::{Result, bail};
+use bincode::{
+    Decode, Encode,
+    de::Decoder,
+    enc::Encoder,
+    error::{DecodeError, EncodeError},
+};
 use regex::bytes::{Regex, RegexBuilder};
 use serde::{Deserialize, Serialize};
 use turbo_rcstr::{RcStr, rcstr};
@@ -20,8 +26,8 @@ use crate::globset::parse;
 // Note: a/**/b does match a/b, so we need some special logic about path
 // separators
 
-#[turbo_tasks::value(eq = "manual")]
-#[derive(Debug, Clone)]
+#[turbo_tasks::value(eq = "manual", serialization = "custom")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(into = "GlobForm", try_from = "GlobForm")]
 pub struct Glob {
     glob: RcStr,
@@ -46,8 +52,37 @@ impl Display for Glob {
         write!(f, "Glob({})", self.glob)
     }
 }
+
+impl Encode for Glob {
+    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        self.glob.encode(encoder)?;
+        self.opts.encode(encoder)?;
+        Ok(())
+    }
+}
+
+impl<Context> Decode<Context> for Glob {
+    fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
+        let glob = RcStr::decode(decoder)?;
+        let opts = GlobOptions::decode(decoder)?;
+        Glob::parse(glob, opts).map_err(|err| DecodeError::OtherString(err.to_string()))
+    }
+}
+
 #[derive(
-    Serialize, Deserialize, Copy, Clone, PartialEq, Eq, Hash, Default, TaskInput, TraceRawVcs, Debug,
+    Serialize,
+    Deserialize,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    Hash,
+    Default,
+    TaskInput,
+    TraceRawVcs,
+    Debug,
+    Encode,
+    Decode,
 )]
 
 pub struct GlobOptions {