Skip to content
2 changes: 1 addition & 1 deletion crates/composefs-integration-tests/src/tests/privileged.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ impl Drop for FuseOverlayTempDir {
/// support `O_TMPFILE`, ingesting object data used to fail outright with
/// `EOPNOTSUPP` ("Opening temp file in objects directory"). This is
/// exactly the path bootc's composefs digest computation exercises
/// (`ensure_object_from_reader`, via `ensure_object_from_fd`) when reading
/// (`ensure_object_from_fd`) when reading
/// a container rootfs through a storage backend without O_TMPFILE support.
///
/// Verifies the repository's named-tmpfile fallback lets object ingestion
Expand Down
2 changes: 1 addition & 1 deletion crates/composefs-oci/src/delta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ fn reconstruct_layer<ObjectID: FsVerityHashValue>(
expected_diff_id: &OciDigest,
) -> Result<File> {
let tmpfile_fd = repo
.create_object_tmpfile()
.create_scratch_tmpfile()
.context("Creating temp file for layer reconstruction")?;
let mut tmpfile = File::from(tmpfile_fd);
let mut data_source = ComposeFsDataSource {
Expand Down
11 changes: 6 additions & 5 deletions crates/composefs-oci/src/layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,21 +93,22 @@ where
/// Use this for non-tar blobs (OCI artifacts) where the caller will create
/// the splitstream wrapper.
///
/// The blob must be exactly `size` bytes long.
///
/// Returns (object_id, size, store_method) of the stored object.
pub async fn store_blob_async<ObjectID, R>(
repo: &Repository<ObjectID>,
mut reader: R,
size: u64,
) -> Result<(ObjectID, u64, ObjectStoreMethod)>
where
ObjectID: FsVerityHashValue,
R: AsyncRead + Unpin,
{
let tmpfile = repo.create_object_tmpfile()?;
let mut writer = tokio::fs::File::from(std::fs::File::from(tmpfile));
let size = tokio::io::copy(&mut reader, &mut writer).await?;
let mut writer = repo.create_object(size)?;
tokio::io::copy(&mut reader, &mut writer).await?;
writer.flush().await?;
let tmpfile = writer.into_std().await;
let (object_id, method) = repo.finalize_object_tmpfile(tmpfile, size)?;
let (object_id, method) = writer.finish_async().await?;
Ok((object_id, size, method))
}

Expand Down
5 changes: 4 additions & 1 deletion crates/composefs-oci/src/oci_layout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,10 @@ async fn import_layer_from_blob<ObjectID: FsVerityHashValue>(
} else {
// Non-tar blob: store as object and create splitstream wrapper.
// Run the progress driver concurrently with the blob store.
let (store_result, ()) = tokio::join!(store_blob_async(repo, async_file), progress_driver);
let (store_result, ()) = tokio::join!(
store_blob_async(repo, async_file, layer_size),
progress_driver
);
let (object_id, size, method) = store_result?;

let mut stats = ImportStats::default();
Expand Down
8 changes: 5 additions & 3 deletions crates/composefs-oci/src/skopeo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,10 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
} else {
// Non-tar layers (OCI artifacts): stream raw bytes to object store.
// Run the progress driver concurrently with the blob store.
let (store_result, ()) =
tokio::join!(store_blob_async(&self.repo, blob_reader), progress_driver);
let (store_result, ()) = tokio::join!(
store_blob_async(&self.repo, blob_reader, descriptor.size()),
progress_driver
);
let (object_id, size, method) = store_result?;
driver.await?;

Expand Down Expand Up @@ -556,7 +558,7 @@ impl<ObjectID: FsVerityHashValue> oci_delta::DeltaBlobReader for ProxyBlobReader
let tmpfile = self
.image_op
.repo
.create_object_tmpfile()
.create_scratch_tmpfile()
.context("Creating temp file for delta blob")?;
let copy_fut = async {
let mut async_dst = tokio::fs::File::from(std::fs::File::from(tmpfile));
Expand Down
14 changes: 4 additions & 10 deletions crates/composefs-oci/src/tar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use std::{
collections::BTreeMap,
ffi::{OsStr, OsString},
fmt,
fs::File,
os::unix::prelude::{OsStrExt, OsStringExt},
path::PathBuf,
sync::Arc,
Expand Down Expand Up @@ -85,21 +84,16 @@ fn receive_and_finalize_object<ObjectID: FsVerityHashValue>(
) -> Result<(ObjectID, ObjectStoreMethod)> {
use std::io::Write;

// Create tmpfile in the blocking context
let tmpfile_fd = repo.create_object_tmpfile()?;
let mut tmpfile = std::io::BufWriter::with_capacity(IO_BUF_CAPACITY, File::from(tmpfile_fd));
let mut writer = repo.create_object(size)?;

// Receive chunks and write to tmpfile
// Receive chunks and write them to the object
let mut rx = rx;
while let Some(chunk) = rx.blocking_recv() {
tmpfile.write_all(&chunk)?;
writer.write_all(&chunk)?;
}

// Flush and get the File back
let tmpfile = tmpfile.into_inner()?;

// Finalize: enable verity, get digest, link into objects/
repo.finalize_object_tmpfile(tmpfile, size)
writer.finish()
}

/// Stream a large file's content through a channel to a background storage task.
Expand Down
5 changes: 3 additions & 2 deletions crates/composefs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ serde = { version = "1.0", default-features = false, features = ["derive", "allo
serde_repr = "0.1"
fn-error-context = "0.2"
hex = { version = "0.4.0", default-features = false, features = ["std"] }
libc = "0.2"
log = { version = "0.4.8", default-features = false }
once_cell = { version = "1.21.3", default-features = false, features = ["std"] }
rustix = { version = "1.0.0", default-features = false, features = ["fs", "mount", "process", "std", "thread"] }
rustix = { version = "1.0.0", default-features = false, features = ["event", "fs", "mount", "net", "pipe", "process", "rand", "std", "thread"] }
serde_json = { version = "1.0", default-features = false, features = ["std"] }
openssl = { version = "0.10", default-features = false }
thiserror = { version = "2.0.0", default-features = false }
tokio = { version = "1.24.2", default-features = false, features = ["macros", "process", "io-util", "rt-multi-thread", "sync"] }
tokio = { version = "1.36.0", default-features = false, features = ["macros", "net", "process", "io-util", "rt-multi-thread", "sync"] }
tempfile = { version = "3.8.0", optional = true, default-features = false }
xxhash-rust = { version = "0.8.2", default-features = false, features = ["xxh32"] }
zerocopy = { version = "0.8.0", default-features = false, features = ["derive", "std"] }
Expand Down
161 changes: 77 additions & 84 deletions crates/composefs/src/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ use zerocopy::IntoBytes;

use crate::{
INLINE_CONTENT_MAX_V0,
fsverity::{FsVerityHashValue, FsVerityHasher},
fsverity::{FsVerityHashValue, FsVerityHasher, FsVerityHelper},
generic_tree,
generic_tree::OciTransformOptions,
repository::Repository,
shared_internals::IO_BUF_CAPACITY,
tree::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat},
util::{create_tmpfile_in, proc_self_fd, reopen_tmpfile_ro},
util::proc_self_fd,
};

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -86,6 +86,8 @@ pub struct FlatDigestStore {
/// Open directory fd for the store root.
root: Arc<OwnedFd>,
semaphore: Arc<Semaphore>,
/// Creates object files with fs-verity enabled.
helper: FsVerityHelper,
/// If true, fall back to userspace hashing when kernel fs-verity is
/// unavailable (e.g. tmpfs, overlayfs). Matches `Repository::insecure`.
insecure: bool,
Expand Down Expand Up @@ -117,82 +119,64 @@ impl FlatDigestStore {
)
.with_context(|| format!("Failed to open flat digest store: {path:?}"))?;

let helper = FsVerityHelper::builder()
.max_workers(concurrency)
.userspace_digest(insecure)
.start()
.context("Starting fs-verity helper")?;

Ok(Self {
root: Arc::new(root),
semaphore: Arc::new(Semaphore::new(concurrency)),
helper,
insecure,
})
}
}

impl<ObjectID: FsVerityHashValue> ObjectStore<ObjectID> for FlatDigestStore {
fn ensure_object_from_fd(&self, fd: OwnedFd, size: u64) -> Result<ObjectID> {
use crate::fsverity::{EnableVerityError, enable_verity_maybe_copy, measure_verity};
use std::io::BufRead as _;

// 1. Create an anonymous O_TMPFILE in the store root.
// No name collision possible; invisible until linked.
let tmpfile_fd = create_tmpfile_in(self.root.as_fd())
.context("Creating O_TMPFILE in flat digest store")?;

// 2. Stream from source fd into tmpfile (no in-memory buffering).
let mut src = std::io::BufReader::with_capacity(IO_BUF_CAPACITY, File::from(fd));
let mut dst = File::from(tmpfile_fd.try_clone().context("Cloning tmpfile fd")?);
let copied = std::io::copy(&mut src, &mut dst).context("Copying object data to tmpfile")?;
ensure!(
copied == size,
"object size mismatch: expected {size}, copied {copied}"
);
drop(dst);

// 3. Reopen as read-only (kernel requires no writable fds to enable verity).
let ro_fd =
reopen_tmpfile_ro(File::from(tmpfile_fd)).context("Reopening tmpfile as read-only")?;

// 4. Enable kernel fs-verity (kernel reads and hashes the file for us).
let (ro_fd, verity_enabled) =
match enable_verity_maybe_copy::<ObjectID>(self.root.as_fd(), ro_fd.as_fd()) {
Ok(None) => (ro_fd, true),
Ok(Some(new_fd)) => (new_fd, true),
Err(EnableVerityError::AlreadyEnabled) => (ro_fd, true),
Err(EnableVerityError::FilesystemNotSupported) if self.insecure => (ro_fd, false),
Err(e) => {
return Err(anyhow::anyhow!(e)).context("Enabling verity on object tmpfile");
}
};

// 5. Get the digest — from the kernel (fast) or userspace fallback.
let id: ObjectID = if verity_enabled {
measure_verity(&ro_fd).context("Measuring verity digest after enable")?
} else {
// Insecure fallback: re-read the tmpfile to compute the digest.
let mut reader = std::io::BufReader::with_capacity(
IO_BUF_CAPACITY,
File::from(ro_fd.try_clone().context("Cloning ro_fd for digest")?),
);
let mut hasher = FsVerityHasher::<ObjectID>::new();
loop {
let buf = reader.fill_buf().context("Reading tmpfile for digest")?;
if buf.is_empty() {
break;
}
let chunk = &buf[..buf.len().min(FsVerityHasher::<ObjectID>::BLOCK_SIZE)];
hasher.add_block(chunk);
let n = chunk.len();
reader.consume(n);
use crate::fsverity::{FinishError, FinishedFile};

// 1. Copy the data into a new file with fs-verity enabled (by
// reflink if possible), via the helper so no writable
// descriptor to it can leak into a forked child.
let result = self
.helper
.create_from_fd::<ObjectID>(self.root.as_fd(), &fd, size);
drop(fd);

// 2. Get the digest: from the kernel, or (insecure mode, without
// fs-verity support) computed in userspace.
let (file, id) = match result {
Ok(file) => {
let id = file
.digest::<ObjectID>()
.context("fs-verity digest has the wrong algorithm")?;
(FinishedFile::Verity(file), id)
}
hasher.digest()
Err(FinishError::NotSupported(file)) if self.insecure => {
let id = match file.userspace_digest::<ObjectID>() {
Some(id) => id,
None => compute_verity_from_fd(
file.as_fd()
.try_clone_to_owned()
.context("Cloning fd for digest")?,
)?,
};
(FinishedFile::Unverified(file), id)
}
Err(e) => return Err(e).context("Creating object in flat digest store"),
};

// 6. Derive flat path: XX/rest-of-hex (C-compatible layout).
// 3. Derive flat path: XX/rest-of-hex (C-compatible layout).
let obj_path = id.to_object_pathname();
let slash = obj_path
.find('/')
.expect("to_object_pathname always has '/'");
let dir_name = &obj_path[..slash];
let file_name = &obj_path[slash + 1..];

// 7. Create XX/ subdirectory if needed.
// 4. Create XX/ subdirectory if needed.
match mkdirat(self.root.as_fd(), dir_name, Mode::from_raw_mode(0o755)) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(e) => {
Expand All @@ -201,31 +185,10 @@ impl<ObjectID: FsVerityHashValue> ObjectStore<ObjectID> for FlatDigestStore {
}
}

// 8. Open the XX/ subdirectory for use as linkat target.
let subdir = openat(
self.root.as_fd(),
dir_name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC,
Mode::empty(),
)
.with_context(|| format!("Opening digest store subdirectory {dir_name:?}"))?;

// 9. Atomically link the tmpfile into its final content-addressed path.
// EEXIST means another writer already stored the same object — fine.
match linkat(
CWD,
proc_self_fd(&ro_fd),
&subdir,
file_name,
AtFlags::SYMLINK_FOLLOW,
) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(e) => {
return Err(e).with_context(|| {
format!("Linking object into flat digest store: {obj_path:?}")
});
}
}
// 5. Atomically put the file at its final content-addressed path.
// If it already exists, another writer stored the same object.
file.publish(self.root.as_fd(), &obj_path)
.with_context(|| format!("Linking object into flat digest store: {obj_path:?}"))?;

Ok(id)
}
Expand Down Expand Up @@ -1057,6 +1020,36 @@ mod tests {
Ok(())
}

/// A secure `FlatDigestStore` gives its objects fs-verity.
#[test]
fn test_flat_digest_store_verity() -> Result<()> {
use crate::fsverity::{Sha256HashValue, compute_verity, measure_verity};

let td = crate::test::tempdir();
let store_path = td.path().join("store");
let store = FlatDigestStore::open(&store_path, 2, false)?;

let content = b"protected by fs-verity".repeat(100);
let src_path = td.path().join("file");
std::fs::write(&src_path, &content)?;
let src_fd = openat(
CWD,
&src_path,
OFlags::RDONLY | OFlags::CLOEXEC,
Mode::empty(),
)?;
let id = <FlatDigestStore as ObjectStore<Sha256HashValue>>::ensure_object_from_fd(
&store,
src_fd,
content.len() as u64,
)?;
assert_eq!(id, compute_verity::<Sha256HashValue>(&content));

let stored = File::open(store_path.join(id.to_object_pathname()))?;
assert_eq!(measure_verity::<Sha256HashValue>(&stored)?, id);
Ok(())
}

/// Verify that `FlatDigestStore` stores objects in the C-compatible `XX/DIGEST` layout.
#[test]
fn test_flat_digest_store_layout() -> Result<()> {
Expand Down
Loading
Loading