Create actual tar.zstd archive

commit-id:1b03d139

Cargo.toml

@@ -79,6 +79,7 @@ similar-asserts = { version = "1.5.0", features = ["serde"] }
 smallvec = "1.11.1"
 smol_str = { version = "0.2.0", features = ["serde"] }
 snapbox = { version = "0.4.12", features = ["cmd", "path"] }
+tar = "0.4.40"
 tempfile = "3.8.0"
 test-case = "3.2.1"
 thiserror = "1.0.48"
@@ -97,6 +98,7 @@ windows-sys = "0.48.0"
 xshell = "0.2.5"
 xxhash-rust = { version = "0.8.7", features = ["xxh3"] }
 zip = { version = "0.6.6", default-features = false, features = ["deflate"] }
+zstd = "0.12.4"
 
 [profile.release]
 lto = true

scarb/Cargo.toml

@@ -57,6 +57,7 @@ serde.workspace = true
 serde_json.workspace = true
 smallvec.workspace = true
 smol_str.workspace = true
+tar.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 toml.workspace = true
@@ -71,6 +72,7 @@ which.workspace = true
 windows-sys.workspace = true
 xxhash-rust.workspace = true
 zip.workspace = true
+zstd.workspace = true
 
 [dev-dependencies]
 assert_fs.workspace = true

scarb/src/core/package/id.rs

@@ -90,6 +90,18 @@ impl PackageId {
             self.source_id.to_pretty_url(),
         )
     }
+
+    /// Basename of the tarball that would be created for this package, e.g. `foo-1.2.3`.
+    pub fn tarball_basename(&self) -> String {
+        format!("{}-{}", self.name, self.version)
+    }
+
+    /// Filename of the tarball that would be created for this package, e.g. `foo-1.2.3.tar.zstd`.
+    pub fn tarball_name(&self) -> String {
+        let mut base = self.tarball_basename();
+        base.push_str(".tar.zstd");
+        base
+    }
 }
 
 impl Deref for PackageId {
@@ -211,4 +223,13 @@ mod tests {
         let pkg_id = PackageId::new(name, version, source_id);
         assert_eq!(format!("foo v1.0.0 ({source_id})"), pkg_id.to_string());
     }
+
+    #[test]
+    fn tarball_name() {
+        let name = PackageName::new("foo");
+        let version = Version::new(1, 0, 0);
+        let source_id = SourceId::mock_path();
+        let pkg_id = PackageId::new(name, version, source_id);
+        assert_eq!("foo-1.0.0.tar.zstd", pkg_id.tarball_name());
+    }
 }

scarb/src/internal/fsx.rs

@@ -73,6 +73,15 @@ pub fn read_to_string(path: impl AsRef<Path>) -> Result<String> {
     }
 }
 
+/// Equivalent to [`fs::rename`] with better error messages.
+pub fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
+    return inner(from.as_ref(), to.as_ref());
+
+    fn inner(from: &Path, to: &Path) -> Result<()> {
+        fs::rename(from, to).with_context(|| format!("failed to rename file: {}", from.display()))
+    }
+}
+
 pub trait PathUtf8Ext {
     fn try_as_utf8(&'_ self) -> Result<&'_ Utf8Path>;
 

scarb/src/ops/package.rs

@@ -1,12 +1,16 @@
 use std::collections::BTreeMap;
+use std::fs::File;
+use std::io::{Seek, SeekFrom};
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context, Result};
 use camino::Utf8PathBuf;
 
 use scarb_ui::components::Status;
+use scarb_ui::{HumanBytes, HumanCount};
 
 use crate::core::{Package, PackageId, PackageName, Workspace};
 use crate::flock::FileLockGuard;
+use crate::internal::fsx;
 use crate::{ops, DEFAULT_SOURCE_PATH, MANIFEST_FILE_NAME};
 
 const VERSION: u8 = 1;
@@ -98,13 +102,53 @@ fn package_one_impl(
 ) -> Result<FileLockGuard> {
     let pkg = ws.fetch_package(&pkg_id)?;
 
+    ws.config()
+        .ui()
+        .print(Status::new("Packaging", &pkg_id.to_string()));
+
     // TODO(mkaput): Check metadata
 
     // TODO(#643): Check dirty in VCS (but do not do it when listing!).
 
-    let _recipe = prepare_archive_recipe(pkg, ws)?;
+    let recipe = prepare_archive_recipe(pkg, ws)?;
+    let num_files = recipe.len();
+
+    // Package up and test a temporary tarball and only move it to the final location if it actually
+    // passes all verification checks. Any previously existing tarball can be assumed as corrupt
+    // or invalid, so we can overwrite it if it exists.
+    let filename = pkg_id.tarball_name();
+    let target_dir = ws.target_dir().child("package");
+
+    let mut dst =
+        target_dir.open_rw(format!(".{filename}"), "package scratch space", ws.config())?;
+
+    dst.set_len(0)
+        .with_context(|| format!("failed to truncate: {filename}"))?;
+
+    let uncompressed_size = tar(pkg_id, recipe, &mut dst, ws)?;
+
+    // TODO(mkaput): Verify.
+
+    dst.seek(SeekFrom::Start(0))?;
+
+    fsx::rename(dst.path(), dst.path().with_file_name(filename))?;
 
-    todo!("Actual packaging is not implemented yet.")
+    let dst_metadata = dst
+        .metadata()
+        .with_context(|| format!("failed to stat: {}", dst.path()))?;
+    let compressed_size = dst_metadata.len();
+
+    ws.config().ui().print(Status::new(
+        "Packaged",
+        &format!(
+            "{} files, {:.1} ({:.1} compressed)",
+            HumanCount(num_files as u64),
+            HumanBytes(uncompressed_size),
+            HumanBytes(compressed_size),
+        ),
+    ));
+
+    Ok(dst)
 }
 
 fn list_one_impl(
@@ -192,3 +236,72 @@ fn normalize_manifest(_pkg: &Package, _ws: &Workspace<'_>) -> Result<Vec<u8>> {
     // TODO(mkaput): Implement this properly.
     Ok("[package]".to_string().into_bytes())
 }
+
+/// Compress and package the recipe, and write it into the given file.
+///
+/// Returns the uncompressed size of the contents of the archive.
+fn tar(
+    pkg_id: PackageId,
+    recipe: ArchiveRecipe<'_>,
+    dst: &mut File,
+    ws: &Workspace<'_>,
+) -> Result<u64> {
+    const COMPRESSION_LEVEL: i32 = 22;
+    let encoder = zstd::stream::Encoder::new(dst, COMPRESSION_LEVEL)?;
+    let mut ar = tar::Builder::new(encoder);
+
+    let base_path = Utf8PathBuf::from(pkg_id.tarball_basename());
+
+    let mut uncompressed_size = 0;
+    for ArchiveFile { path, contents } in recipe {
+        ws.config()
+            .ui()
+            .verbose(Status::new("Archiving", path.as_str()));
+
+        let archive_path = base_path.join(&path);
+        let mut header = tar::Header::new_gnu();
+        match contents {
+            ArchiveFileContents::OnDisk(disk_path) => {
+                let mut file = File::open(&disk_path)
+                    .with_context(|| format!("failed to open for archiving: {disk_path}"))?;
+
+                let metadata = file
+                    .metadata()
+                    .with_context(|| format!("failed to stat: {disk_path}"))?;
+
+                header.set_metadata_in_mode(&metadata, tar::HeaderMode::Deterministic);
+                header.set_cksum();
+
+                ar.append_data(&mut header, &archive_path, &mut file)
+                    .with_context(|| format!("could not archive source file: {disk_path}"))?;
+
+                uncompressed_size += metadata.len();
+            }
+
+            ArchiveFileContents::Generated(generator) => {
+                let contents = generator()?;
+
+                header.set_entry_type(tar::EntryType::file());
+                header.set_mode(0o644);
+                header.set_size(contents.len() as u64);
+
+                // From `set_metadata_in_mode` implementation in `tar` crate:
+                // We could in theory set the mtime to zero here, but not all
+                // tools seem to behave well when ingesting files with a 0
+                // timestamp.
+                header.set_mtime(1);
+
+                header.set_cksum();
+
+                ar.append_data(&mut header, &archive_path, contents.as_slice())
+                    .with_context(|| format!("could not archive source file: {path}"))?;
+
+                uncompressed_size += contents.len() as u64;
+            }
+        }
+    }
+
+    let encoder = ar.into_inner()?;
+    encoder.finish()?;
+    Ok(uncompressed_size)
+}