diff --git a/crates/cfsctl/src/lib.rs b/crates/cfsctl/src/lib.rs index 65856d31..3e708f80 100644 --- a/crates/cfsctl/src/lib.rs +++ b/crates/cfsctl/src/lib.rs @@ -25,7 +25,7 @@ pub use composefs_oci; use std::{ ffi::OsString, fs::create_dir_all, - io::IsTerminal, + io::{IsTerminal, Read}, path::{Path, PathBuf}, sync::Arc, }; @@ -39,9 +39,13 @@ use rustix::fs::CWD; use composefs_boot::{write_boot, BootOps}; use composefs::{ + dumpfile::{dump_single_dir, dump_single_file}, + erofs::reader::erofs_to_filesystem, fsverity::{FsVerityHashValue, Sha256HashValue, Sha512HashValue}, + generic_tree::{FileSystem, Inode}, repository::Repository, shared_internals::IO_BUF_CAPACITY, + tree::RegularFile, }; /// cfsctl @@ -72,8 +76,8 @@ pub struct App { cmd: Command, } +/// The Hash algorithm used for FsVerity computation #[derive(Debug, Copy, Clone, PartialEq, Eq, ValueEnum, Default)] -/// TODO: Hash type pub enum HashType { /// Sha256 Sha256, @@ -95,7 +99,8 @@ struct OCIConfigFilesystemOptions { /// Common options for operations using OCI config manifest streams #[derive(Debug, Parser)] struct OCIConfigOptions { - /// the name of the target OCI manifest stream, either a stream ID in format oci-config-: or a reference in 'ref/' + /// the name of the target OCI manifest stream, + /// either a stream ID in format oci-config-: or a reference in 'ref/' config_name: String, /// verity digest for the manifest stream to be verified against config_verity: Option, @@ -293,6 +298,20 @@ enum Command { /// the name of the image to read, either an object ID digest or prefixed with 'ref/' name: String, }, + /// Extract file information from a composefs image for specified files or directories + /// + /// By default, outputs information in composefs dumpfile format + DumpFiles { + /// The name of the composefs image to read from, either an object ID digest or prefixed with 'ref/' + image_name: String, + /// File or directory paths to process. If a path is a directory, its contents will be listed. + files: Vec, + /// Show backing path information instead of dumpfile format + /// For each file, prints either "inline" for files stored within the image, + /// or a path relative to the object store for files stored extrenally + #[clap(long)] + backing_path_only: bool, + }, #[cfg(feature = "http")] Fetch { url: String, name: String }, } @@ -349,6 +368,94 @@ where Ok(repo) } +fn load_filesystem_from_oci_image( + repo: &Repository, + opts: OCIConfigFilesystemOptions, +) -> Result>> { + let verity = verity_opt(&opts.base_config.config_verity)?; + let mut fs = composefs_oci::image::create_filesystem( + repo, + &opts.base_config.config_name, + verity.as_ref(), + )?; + if opts.bootable { + fs.transform_for_boot(repo)?; + } + Ok(fs) +} + +fn load_filesystem_from_ondisk_fs( + fs_opts: &FsReadOptions, + repo: &Repository, +) -> Result>> { + let mut fs = if fs_opts.no_propagate_usr_to_root { + composefs::fs::read_filesystem(CWD, &fs_opts.path, Some(repo))? + } else { + composefs::fs::read_container_root(CWD, &fs_opts.path, Some(repo))? + }; + if fs_opts.bootable { + fs.transform_for_boot(repo)?; + } + Ok(fs) +} + +fn dump_file_impl( + fs: FileSystem>, + files: &Vec, + backing_path_only: bool, +) -> Result<()> { + let mut out = Vec::new(); + + for file_path in files { + let (dir, file) = fs.root.split(file_path.as_os_str())?; + + let (_, file) = dir + .entries() + .find(|ent| ent.0 == file) + .ok_or_else(|| anyhow::anyhow!("{} not found", file_path.display()))?; + + match &file { + Inode::Directory(directory) => { + if backing_path_only { + anyhow::bail!("{} is a directory", file_path.display()); + } + + dump_single_dir(&mut out, directory, file_path.clone())? + } + + Inode::Leaf(leaf) => { + use composefs::generic_tree::LeafContent::*; + use composefs::tree::RegularFile::*; + + if backing_path_only { + match &leaf.content { + Regular(f) => match f { + Inline(..) => println!("{} inline", file_path.display()), + External(id, _) => { + println!("{} {}", file_path.display(), id.to_object_pathname()); + } + }, + _ => { + println!("{} inline", file_path.display()) + } + } + + continue; + } + + dump_single_file(&mut out, leaf, file_path.clone())? + } + }; + } + + if !out.is_empty() { + let out_str = std::str::from_utf8(&out).unwrap(); + println!("{}", out_str); + } + + Ok(()) +} + /// Run with cmd pub async fn run_cmd_with_repo(repo: Repository, args: App) -> Result<()> where @@ -384,63 +491,20 @@ where OciCommand::LsLayer { name } => { composefs_oci::ls_layer(&repo, &name)?; } - OciCommand::Dump { - config_opts: - OCIConfigFilesystemOptions { - base_config: - OCIConfigOptions { - ref config_name, - ref config_verity, - }, - bootable, - }, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - if bootable { - fs.transform_for_boot(&repo)?; - } + OciCommand::Dump { config_opts } => { + let fs = load_filesystem_from_oci_image(&repo, config_opts)?; fs.print_dumpfile()?; } - OciCommand::ComputeId { - config_opts: - OCIConfigFilesystemOptions { - base_config: - OCIConfigOptions { - ref config_name, - ref config_verity, - }, - bootable, - }, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - if bootable { - fs.transform_for_boot(&repo)?; - } + OciCommand::ComputeId { config_opts } => { + let fs = load_filesystem_from_oci_image(&repo, config_opts)?; let id = fs.compute_image_id(); println!("{}", id.to_hex()); } OciCommand::CreateImage { - config_opts: - OCIConfigFilesystemOptions { - base_config: - OCIConfigOptions { - ref config_name, - ref config_verity, - }, - bootable, - }, + config_opts, ref image_name, } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - if bootable { - fs.transform_for_boot(&repo)?; - } + let fs = load_filesystem_from_oci_image(&repo, config_opts)?; let image_id = fs.commit_image(&repo, image_name.as_deref())?; println!("{}", image_id.to_id()); } @@ -627,14 +691,7 @@ where } }, Command::ComputeId { fs_opts } => { - let mut fs = if fs_opts.no_propagate_usr_to_root { - composefs::fs::read_filesystem(CWD, &fs_opts.path, Some(&repo))? - } else { - composefs::fs::read_container_root(CWD, &fs_opts.path, Some(&repo))? - }; - if fs_opts.bootable { - fs.transform_for_boot(&repo)?; - } + let fs = load_filesystem_from_ondisk_fs(&fs_opts, &repo)?; let id = fs.compute_image_id(); println!("{}", id.to_hex()); } @@ -642,26 +699,12 @@ where fs_opts, ref image_name, } => { - let mut fs = if fs_opts.no_propagate_usr_to_root { - composefs::fs::read_filesystem(CWD, &fs_opts.path, Some(&repo))? - } else { - composefs::fs::read_container_root(CWD, &fs_opts.path, Some(&repo))? - }; - if fs_opts.bootable { - fs.transform_for_boot(&repo)?; - } + let fs = load_filesystem_from_ondisk_fs(&fs_opts, &repo)?; let id = fs.commit_image(&repo, image_name.as_deref())?; println!("{}", id.to_id()); } Command::CreateDumpfile { fs_opts } => { - let mut fs = if fs_opts.no_propagate_usr_to_root { - composefs::fs::read_filesystem(CWD, &fs_opts.path, Some(&repo))? - } else { - composefs::fs::read_container_root(CWD, &fs_opts.path, Some(&repo))? - }; - if fs_opts.bootable { - fs.transform_for_boot(&repo)?; - } + let fs = load_filesystem_from_ondisk_fs(&fs_opts, &repo)?; fs.print_dumpfile()?; } Command::Mount { name, mountpoint } => { @@ -694,6 +737,30 @@ where ); } } + Command::DumpFiles { + image_name, + files, + backing_path_only, + } => { + let (img_fd, _) = repo.open_image(&image_name)?; + + let mut img_buf = Vec::new(); + std::fs::File::from(img_fd).read_to_end(&mut img_buf)?; + + match args.hash { + HashType::Sha256 => dump_file_impl( + erofs_to_filesystem::(&img_buf)?, + &files, + backing_path_only, + )?, + + HashType::Sha512 => dump_file_impl( + erofs_to_filesystem::(&img_buf)?, + &files, + backing_path_only, + )?, + }; + } #[cfg(feature = "http")] Command::Fetch { url, name } => { let (digest, verity) = composefs_http::download(&url, &name, Arc::new(repo)).await?; diff --git a/crates/composefs/src/dumpfile.rs b/crates/composefs/src/dumpfile.rs index 4a9fcf26..88159bcc 100644 --- a/crates/composefs/src/dumpfile.rs +++ b/crates/composefs/src/dumpfile.rs @@ -347,14 +347,40 @@ impl<'a, W: Write, ObjectID: FsVerityHashValue> DumpfileWriter<'a, W, ObjectID> pub fn write_dumpfile( writer: &mut impl Write, fs: &FileSystem, +) -> Result<()> { + let path = PathBuf::from("/"); + dump_single_dir(writer, &fs.root, path) +} + +/// Write a single dir +pub fn dump_single_dir( + writer: &mut impl Write, + dir: &Directory, + mut path: PathBuf, +) -> Result<()> { + // default pipe capacity on Linux is 16 pages (65536 bytes), but + // sometimes the BufWriter will write more than its capacity... + let mut buffer = BufWriter::with_capacity(32768, writer); + let mut dfw = DumpfileWriter::new(&mut buffer); + + dfw.write_dir(&mut path, dir)?; + buffer.flush()?; + + Ok(()) +} + +/// Write a single file +pub fn dump_single_file( + writer: &mut impl Write, + file: &Rc>, + path: PathBuf, ) -> Result<()> { // default pipe capacity on Linux is 16 pages (65536 bytes), but // sometimes the BufWriter will write more than its capacity... let mut buffer = BufWriter::with_capacity(32768, writer); let mut dfw = DumpfileWriter::new(&mut buffer); - let mut path = PathBuf::from("/"); - dfw.write_dir(&mut path, &fs.root)?; + dfw.write_leaf(&path, file)?; buffer.flush()?; Ok(()) diff --git a/crates/composefs/src/repository.rs b/crates/composefs/src/repository.rs index fb21c848..908717b7 100644 --- a/crates/composefs/src/repository.rs +++ b/crates/composefs/src/repository.rs @@ -877,7 +877,7 @@ impl Repository { /// Returns the fd of the image and whether or not verity should be /// enabled when mounting it. #[context("Opening image '{name}'")] - fn open_image(&self, name: &str) -> Result<(OwnedFd, bool)> { + pub fn open_image(&self, name: &str) -> Result<(OwnedFd, bool)> { let image = self .openat(&format!("images/{name}"), OFlags::RDONLY) .with_context(|| format!("Opening ref 'images/{name}'"))?; diff --git a/crates/integration-tests/src/tests/cli.rs b/crates/integration-tests/src/tests/cli.rs index eb954f26..f524ff15 100644 --- a/crates/integration-tests/src/tests/cli.rs +++ b/crates/integration-tests/src/tests/cli.rs @@ -6,6 +6,7 @@ //! smoke tests. use anyhow::Result; +use rustix::path::Arg; use xshell::{cmd, Shell}; use crate::{cfsctl, create_test_rootfs, integration_test}; @@ -454,3 +455,65 @@ fn test_oci_layer_inspect() -> Result<()> { Ok(()) } integration_test!(test_oci_layer_inspect); + +fn test_dump_files() -> Result<()> { + let sh = Shell::new()?; + let cfsctl = cfsctl()?; + let repo_dir = tempfile::tempdir()?; + let repo = repo_dir.path(); + let fixture_dir = tempfile::tempdir()?; + let rootfs = create_test_rootfs(fixture_dir.path())?; + + cmd!( + sh, + "{cfsctl} --insecure --repo {repo} create-image {rootfs} my-image" + ) + .read()?; + + // should be of the form /usr/bin/hello + let output = cmd!( + sh, + "{cfsctl} --insecure --repo {repo} dump-files refs/my-image /usr/bin/hello --backing-path-only" + ) + .read()?; + + let path = output.split_whitespace().nth(1).unwrap(); + + assert!( + path != "inline", + "usr/bin/hello should've been large enough to be stored in objects directory" + ); + + let path = path.strip_prefix("/").unwrap_or(path); + + let full_path = repo.join("objects").join(path); + + assert!(full_path.exists()); + + let file_hash = cmd!(sh, "sha512sum") + .arg(rootfs.join("usr/bin/hello").as_str()?) + .read()?; + + let file_hash = file_hash.split_whitespace().next().unwrap().trim(); + + let obj_file_hash = cmd!(sh, "sha512sum").arg(full_path.as_str()?).read()?; + let obj_file_hash = obj_file_hash.split_whitespace().next().unwrap().trim(); + + assert_eq!(file_hash, obj_file_hash); + + let output = cmd!( + sh, + "{cfsctl} --insecure --repo {repo} dump-files refs/my-image /usr/lib/readme.txt --backing-path-only" + ) + .read()?; + + let path = output.split_whitespace().nth(1).unwrap(); + + assert!( + path == "inline", + "usr/lib/readme.txt should've been stored inline" + ); + + Ok(()) +} +integration_test!(test_dump_files);