use ocidir

Cargo.toml

@@ -15,7 +15,6 @@ anyhow = "1.0.75"
 chrono = { version = "0.4.26", features = ["clock"], default-features = false }
 clap = { version = "4.5.6", features = ["derive"] }
 clap-verbosity-flag = "2.0.0"
-dirs = "5.0.1"
 env_logger = "0.11.3"
 filetime = "0.2.22"
 flate2 = { version = "1.0.24", features = ["zlib"], default-features = false }
@@ -26,8 +25,6 @@ nix = { version = "0.29.0", features = [
     "signal",
     "user",
 ], default-features = false }
-oci-spec = { version = "0.6.3", features = ["image"], default-features = false }
-openssl = "0.10.63"
 pathdiff = "0.2.1"
 pyo3 = { version = "0.22.1", features = ["auto-initialize"] }
 rpm = { version = "0.15.0", default-features = false }
@@ -41,6 +38,7 @@ toml = { version = "0.8.8" }
 url = { version = "2.2.2", features = ["serde"] }
 walkdir = "2.3.2"
 xattr = "1.0.1"
+ocidir = { git = "https://github.com/containers/ocidir-rs" }
 
 [dev-dependencies]
 test-temp-dir = "0.2.2"

README.md

@@ -8,7 +8,11 @@ rpmoci features:
  - **unprivileged** rpmoci can build images in environments without access to a container runtime, and without root access (this relies on the user being able to create [user namespaces](https://www.man7.org/linux/man-pages/man7/user_namespaces.7.html))
  - **small** rpmoci images are built solely from the RPMs you request and their dependencies, so don't contain unnecessary dependencies.
 
+rpmoci is a good fit for containerizing applications - you package your application as an RPM, and then use rpmoci to build a minimal container image from that RPM.
+
 The design of rpmoci is influenced by [apko](https://github.com/chainguard-dev/apko) and [distroless](https://github.com/GoogleContainerTools/distroless) tooling.
+rpmoci is also similar to a smaller [`rpm-ostree compose image`](https://coreos.github.io/rpm-ostree/container/#creating-base-images), with a focus on building microservices.
+
 
 ## Installing
 

src/archive.rs

@@ -0,0 +1,159 @@
+//! Copyright (C) Microsoft Corporation.
+//!
+//! This program is free software: you can redistribute it and/or modify
+//! it under the terms of the GNU General Public License as published by
+//! the Free Software Foundation, either version 3 of the License, or
+//! (at your option) any later version.
+//!
+//! This program is distributed in the hope that it will be useful,
+//! but WITHOUT ANY WARRANTY; without even the implied warranty of
+//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//! GNU General Public License for more details.
+//!
+//! You should have received a copy of the GNU General Public License
+//! along with this program.  If not, see <https://www.gnu.org/licenses/>.
+use anyhow::{Context, Result};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    io::Write,
+    os::unix::{
+        fs::MetadataExt,
+        prelude::{FileTypeExt, OsStrExt},
+    },
+    path::{Path, PathBuf},
+};
+use walkdir::WalkDir;
+
+// https://mgorny.pl/articles/portability-of-tar-features.html#id25
+const PAX_SCHILY_XATTR: &[u8; 13] = b"SCHILY.xattr.";
+
+/// custom implementation of tar-rs's append_dir_all that:
+/// - works around https://github.com/alexcrichton/tar-rs/issues/102 so that security capabilities are preserved
+/// - emulates tar's `--clamp-mtime` option so that any file/dir/symlink mtimes are no later than a specific value
+/// - supports hardlinks
+pub(super) fn append_dir_all_with_xattrs(
+    builder: &mut tar::Builder<impl Write>,
+    src_path: impl AsRef<Path>,
+    clamp_mtime: i64,
+) -> Result<()> {
+    let src_path = src_path.as_ref();
+    // Map (dev, inode) -> path for hardlinks
+    let mut hardlinks: HashMap<(u64, u64), PathBuf> = HashMap::new();
+
+    for entry in WalkDir::new(src_path)
+        .follow_links(false)
+        .into_iter()
+        .filter_map(Result::ok)
+    {
+        let meta = entry.metadata()?;
+        // skip sockets as tar-rs errors when trying to archive them.
+        // For comparison, umoci also errors, whereas docker skips them
+        if meta.file_type().is_socket() {
+            continue;
+        }
+
+        let rel_path = pathdiff::diff_paths(entry.path(), src_path)
+            .expect("walkdir returns path inside of search root");
+        if rel_path == Path::new("") {
+            continue;
+        }
+
+        if entry.file_type().is_symlink() {
+            if meta.mtime() > clamp_mtime {
+                // Setting the mtime on a symlink is fiddly with tar-rs, so we use filetime to change
+                // the mtime before adding the symlink to the tar archive
+                let mtime = filetime::FileTime::from_unix_time(clamp_mtime, 0);
+                filetime::set_symlink_file_times(entry.path(), mtime, mtime)?;
+            }
+            add_pax_extension_header(entry.path(), builder)?;
+            builder.append_path_with_name(entry.path(), rel_path)?;
+        } else if entry.file_type().is_file() || entry.file_type().is_dir() {
+            add_pax_extension_header(entry.path(), builder)?;
+
+            // If this is a hardlink, add a link header instead of the file
+            // if this isn't the first time we've seen this inode
+            if meta.nlink() > 1 {
+                match hardlinks.entry((meta.dev(), meta.ino())) {
+                    Entry::Occupied(e) => {
+                        // Add link header and continue to next entry
+                        let mut header = tar::Header::new_gnu();
+                        header.set_metadata(&meta);
+                        if meta.mtime() > clamp_mtime {
+                            header.set_mtime(clamp_mtime as u64);
+                        }
+                        header.set_entry_type(tar::EntryType::Link);
+                        header.set_cksum();
+                        builder.append_link(&mut header, &rel_path, e.get())?;
+                        continue;
+                    }
+                    Entry::Vacant(e) => {
+                        // This is the first time we've seen this inode
+                        e.insert(rel_path.clone());
+                    }
+                }
+            }
+
+            let mut header = tar::Header::new_gnu();
+            header.set_size(meta.len());
+            header.set_metadata(&meta);
+            if meta.mtime() > clamp_mtime {
+                header.set_mtime(clamp_mtime as u64);
+            }
+            if entry.file_type().is_file() {
+                builder.append_data(
+                    &mut header,
+                    rel_path,
+                    &mut std::fs::File::open(entry.path())?,
+                )?;
+            } else {
+                builder.append_data(&mut header, rel_path, &mut std::io::empty())?;
+            };
+        }
+    }
+
+    Ok(())
+}
+
+// Convert any extended attributes on the specified path to a tar PAX extension header, and add it to the tar archive
+fn add_pax_extension_header(
+    path: impl AsRef<Path>,
+    builder: &mut tar::Builder<impl Write>,
+) -> Result<(), anyhow::Error> {
+    let path = path.as_ref();
+    let xattrs = xattr::list(path)
+        .with_context(|| format!("Failed to list xattrs from `{}`", path.display()))?;
+    let mut pax_header = tar::Header::new_ustar();
+    let mut pax_data = Vec::new();
+    for key in xattrs {
+        let value = xattr::get(path, &key)
+            .with_context(|| {
+                format!(
+                    "Failed to get xattr `{}` from `{}`",
+                    key.to_string_lossy(),
+                    path.display()
+                )
+            })?
+            .unwrap_or_default();
+
+        // each entry is "<len> <key>=<value>\n": https://www.ibm.com/docs/en/zos/2.3.0?topic=SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.html
+        let data_len = PAX_SCHILY_XATTR.len() + key.as_bytes().len() + value.len() + 3;
+        // Calculate the total length, including the length of the length field
+        let mut len_len = 1;
+        while data_len + len_len >= 10usize.pow(len_len.try_into().unwrap()) {
+            len_len += 1;
+        }
+        write!(pax_data, "{} ", data_len + len_len)?;
+        pax_data.write_all(PAX_SCHILY_XATTR)?;
+        pax_data.write_all(key.as_bytes())?;
+        pax_data.write_all("=".as_bytes())?;
+        pax_data.write_all(&value)?;
+        pax_data.write_all("\n".as_bytes())?;
+    }
+    if !pax_data.is_empty() {
+        pax_header.set_size(pax_data.len() as u64);
+        pax_header.set_entry_type(tar::EntryType::XHeader);
+        pax_header.set_cksum();
+        builder.append(&pax_header, &*pax_data)?;
+    }
+    Ok(())
+}

src/config.rs

@@ -15,7 +15,7 @@
 //! You should have received a copy of the GNU General Public License
 //! along with this program.  If not, see <https://www.gnu.org/licenses/>.
 use anyhow::Result;
-use oci_spec::{
+use ocidir::oci_spec::{
     image::{Arch, ConfigBuilder, ImageConfiguration, ImageConfigurationBuilder, Os},
     OciSpecError,
 };
@@ -204,11 +204,10 @@ impl ImageConfig {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
-
-    use crate::config::ImageConfig;
-
     use super::Config;
+    use crate::config::ImageConfig;
+    use ocidir::oci_spec::image::ImageConfiguration;
+    use std::collections::HashMap;
 
     #[test]
     fn parse_basic() {
@@ -287,23 +286,21 @@ mod tests {
         let config_with_path = r#"
         envs = { PATH = "/usr/bin"}
         "#;
-        let config: oci_spec::image::ImageConfiguration =
-            toml::from_str::<ImageConfig>(config_with_path)
-                .unwrap()
-                .to_oci_image_configuration(HashMap::new(), chrono::Utc::now())
-                .unwrap();
+        let config: ImageConfiguration = toml::from_str::<ImageConfig>(config_with_path)
+            .unwrap()
+            .to_oci_image_configuration(HashMap::new(), chrono::Utc::now())
+            .unwrap();
         let envs = config.config().as_ref().unwrap().env().as_ref().unwrap();
         assert!(envs.iter().any(|e| e == "PATH=/usr/bin"));
         assert_eq!(envs.len(), 1);
 
         let config_without_path = r#"
         envs = { FOO = "bar"}
         "#;
-        let config: oci_spec::image::ImageConfiguration =
-            toml::from_str::<ImageConfig>(config_without_path)
-                .unwrap()
-                .to_oci_image_configuration(HashMap::new(), chrono::Utc::now())
-                .unwrap();
+        let config: ImageConfiguration = toml::from_str::<ImageConfig>(config_without_path)
+            .unwrap()
+            .to_oci_image_configuration(HashMap::new(), chrono::Utc::now())
+            .unwrap();
         let envs = config.config().as_ref().unwrap().env().as_ref().unwrap();
         assert!(envs
             .iter()
@@ -317,7 +314,7 @@ mod tests {
         labels = { "foo.bar" = "baz"}
         "#;
         // No additional labels
-        let config: oci_spec::image::ImageConfiguration = toml::from_str::<ImageConfig>(config_str)
+        let config: ImageConfiguration = toml::from_str::<ImageConfig>(config_str)
             .unwrap()
             .to_oci_image_configuration(HashMap::new(), chrono::Utc::now())
             .unwrap();
@@ -331,7 +328,7 @@ mod tests {
         ]
         .into_iter()
         .collect();
-        let config: oci_spec::image::ImageConfiguration = toml::from_str::<ImageConfig>(config_str)
+        let config: ImageConfiguration = toml::from_str::<ImageConfig>(config_str)
             .unwrap()
             .to_oci_image_configuration(extra_labels, chrono::Utc::now())
             .unwrap();

src/lib.rs

@@ -22,11 +22,10 @@ use std::{
 };
 
 use anyhow::{bail, Context};
+mod archive;
 pub mod cli;
 pub mod config;
 pub mod lockfile;
-mod oci;
-mod sha256_writer;
 pub mod subid;
 pub mod write;
 use anyhow::Result;