Initial implementation of scarb publish

There are some rough untested edges, but in general this command is capable of building a local registry from scratch 🎉

commit-id:5fd89d54

scarb/src/bin/scarb/args.rs

@@ -11,6 +11,7 @@ use clap::{CommandFactory, Parser, Subcommand};
 use smol_str::SmolStr;
 use tracing::level_filters::LevelFilter;
 use tracing_log::AsTrace;
+use url::Url;
 
 use scarb::compiler::Profile;
 use scarb::core::PackageName;
@@ -156,6 +157,13 @@ pub enum Command {
         codes of selected packages. Resulting files will be placed in `target/package` directory.
     ")]
     Package(PackageArgs),
+    /// Upload a package to the registry.
+    #[command(after_help = "\
+        This command will create distributable, compressed `.tar.zst` archive containing source \
+        code of the package in `target/package` directory (using `scarb package`) and upload it \
+        to a registry.
+    ")]
+    Publish(PublishArgs),
     /// Run arbitrary package scripts.
     Run(ScriptsRunnerArgs),
     /// Execute all unit and integration tests of a local package.
@@ -321,6 +329,17 @@ pub struct PackageArgs {
     pub packages_filter: PackagesFilter,
 }
 
+/// Arguments accepted by the `publish` command.
+#[derive(Parser, Clone, Debug)]
+pub struct PublishArgs {
+    /// Registry index URL to upload the package to.
+    #[arg(long, value_name = "URL")]
+    pub index: Url,
+
+    #[command(flatten)]
+    pub packages_filter: PackagesFilter,
+}
+
 /// Git reference specification arguments.
 #[derive(Parser, Clone, Debug)]
 #[group(requires = "git", multiple = false)]

scarb/src/bin/scarb/commands/mod.rs

@@ -20,6 +20,7 @@ pub mod manifest_path;
 pub mod metadata;
 pub mod new;
 pub mod package;
+pub mod publish;
 pub mod remove;
 pub mod run;
 pub mod test;
@@ -43,6 +44,7 @@ pub fn run(command: Command, config: &mut Config) -> Result<()> {
         Metadata(args) => metadata::run(args, config),
         New(args) => new::run(args, config),
         Package(args) => package::run(args, config),
+        Publish(args) => publish::run(args, config),
         Remove(args) => remove::run(args, config),
         Run(args) => run::run(args, config),
         Test(args) => test::run(args, config),

scarb/src/bin/scarb/commands/publish.rs

@@ -0,0 +1,19 @@
+use anyhow::Result;
+
+use scarb::core::Config;
+use scarb::ops;
+use scarb::ops::PublishOpts;
+
+use crate::args::PublishArgs;
+
+#[tracing::instrument(skip_all, level = "info")]
+pub fn run(args: PublishArgs, config: &Config) -> Result<()> {
+    let ws = ops::read_workspace(config.manifest_path(), config)?;
+    let package = args.packages_filter.match_one(&ws)?;
+
+    let ops = PublishOpts {
+        index_url: args.index,
+    };
+
+    ops::publish(package.id, &ops, &ws)
+}

scarb/src/core/checksum.rs

@@ -1,5 +1,6 @@
 use std::fmt;
 use std::fmt::Write;
+use std::io::Read;
 use std::str;
 use std::str::FromStr;
 
@@ -120,6 +121,17 @@ impl Digest {
         self
     }
 
+    pub fn update_read(&mut self, mut input: impl Read) -> Result<&mut Self> {
+        let mut buf = [0; 64 * 1024];
+        loop {
+            let n = input.read(&mut buf)?;
+            if n == 0 {
+                break Ok(self);
+            }
+            self.update(&buf[..n]);
+        }
+    }
+
     pub fn finish(&mut self) -> Checksum {
         Checksum(self.0.finalize_reset().into())
     }

scarb/src/core/manifest/summary.rs

@@ -87,4 +87,12 @@ impl Summary {
 
         deps.into_iter()
     }
+
+    /// Returns an iterator over dependencies that should be included in registry index record
+    /// for this package.
+    pub fn publish_dependencies(&self) -> impl Iterator<Item = &ManifestDependency> {
+        self.dependencies
+            .iter()
+            .filter(|dep| dep.kind == DepKind::Normal)
+    }
 }

scarb/src/core/package/id.rs

@@ -34,14 +34,18 @@ impl PackageId {
         Self(CACHE.intern(inner))
     }
 
-    pub fn for_test_target(&self, target_name: SmolStr) -> Self {
+    pub fn for_test_target(self, target_name: SmolStr) -> Self {
         Self::new(
             PackageName::new(target_name),
             self.version.clone(),
             self.source_id,
         )
     }
 
+    pub fn with_source_id(self, source_id: SourceId) -> Self {
+        Self::new(self.name.clone(), self.version.clone(), source_id)
+    }
+
     pub fn is_core(&self) -> bool {
         self.name == PackageName::CORE && self.source_id == SourceId::for_std()
     }

scarb/src/core/registry/client/local.rs

@@ -1,15 +1,20 @@
+use std::fs::OpenOptions;
 use std::io;
+use std::io::{BufReader, BufWriter, Seek, SeekFrom};
+use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context, Error, Result};
 use async_trait::async_trait;
+use fs4::FileExt;
 use tokio::task::spawn_blocking;
 use url::Url;
 
 use crate::core::registry::client::RegistryClient;
-use crate::core::registry::index::{IndexRecords, TemplateUrl};
-use crate::core::{PackageId, PackageName};
+use crate::core::registry::index::{IndexDependency, IndexRecord, IndexRecords, TemplateUrl};
+use crate::core::{Checksum, Digest, Package, PackageId, PackageName, Summary};
+use crate::flock::FileLockGuard;
 use crate::internal::fsx;
 
 /// Local registry that lives on the filesystem as a set of `.tar.zst` files with an `index`
@@ -68,6 +73,22 @@ impl LocalRegistryClient {
             dl_template_url,
         })
     }
+
+    fn records_path(&self, package: &PackageName) -> PathBuf {
+        self.index_template_url
+            .expand(package.into())
+            .unwrap()
+            .to_file_path()
+            .unwrap()
+    }
+
+    fn dl_path(&self, package: PackageId) -> PathBuf {
+        self.dl_template_url
+            .expand(package.into())
+            .unwrap()
+            .to_file_path()
+            .unwrap()
+    }
 }
 
 #[async_trait]
@@ -78,11 +99,7 @@ impl RegistryClient for LocalRegistryClient {
 
     #[tracing::instrument(level = "trace", skip(self))]
     async fn get_records(&self, package: PackageName) -> Result<Option<Arc<IndexRecords>>> {
-        let records_path = self
-            .index_template_url
-            .expand(package.into())?
-            .to_file_path()
-            .expect("Local index should always use file:// URLs.");
+        let records_path = self.records_path(&package);
 
         spawn_blocking(move || {
             let records = match fsx::read(records_path) {
@@ -105,10 +122,98 @@ impl RegistryClient for LocalRegistryClient {
     }
 
     async fn download(&self, package: PackageId) -> Result<PathBuf> {
-        Ok(self
-            .dl_template_url
-            .expand(package.into())?
-            .to_file_path()
-            .expect("Local index should always use file:// URLs."))
+        Ok(self.dl_path(package))
+    }
+
+    async fn supports_publish(&self) -> Result<bool> {
+        Ok(true)
+    }
+
+    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<()> {
+        let summary = package.manifest.summary.clone();
+        let records_path = self.records_path(&summary.package_id.name);
+        let dl_path = self.dl_path(summary.package_id);
+
+        spawn_blocking(move || publish_impl(summary, tarball, records_path, dl_path))
+            .await
+            .with_context(|| format!("failed to publish package: {package}"))?
+    }
+}
+
+fn publish_impl(
+    summary: Summary,
+    tarball: FileLockGuard,
+    records_path: PathBuf,
+    dl_path: PathBuf,
+) -> Result<(), Error> {
+    fsx::copy(tarball.path(), dl_path)?;
+
+    let checksum = Digest::recommended().update_read(tarball.deref())?.finish();
+
+    let record = build_record(summary, checksum);
+
+    edit_records(&records_path, move |records| {
+        // Remove existing record if exists (note: version is the key).
+        if let Some(idx) = records.iter().position(|r| r.version == record.version) {
+            records.swap_remove(idx);
+        }
+
+        records.push(record);
+
+        records.sort_by_cached_key(|r| r.version.clone());
+    })
+    .with_context(|| format!("failed to edit records file: {}", records_path.display()))?;
+
+    Ok(())
+}
+
+fn build_record(summary: Summary, checksum: Checksum) -> IndexRecord {
+    IndexRecord {
+        version: summary.package_id.version.clone(),
+        dependencies: summary
+            .publish_dependencies()
+            .map(|dep| IndexDependency {
+                name: dep.name.clone(),
+                req: dep.version_req.clone().into(),
+            })
+            .collect(),
+        checksum,
+        no_core: summary.no_core,
     }
 }
+
+fn edit_records(records_path: &Path, func: impl FnOnce(&mut IndexRecords)) -> Result<()> {
+    fsx::create_dir_all(records_path.parent().unwrap())?;
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(records_path)
+        .context("failed to open file")?;
+
+    file.lock_exclusive()
+        .context("failed to acquire exclusive file access")?;
+
+    let is_empty = file.metadata().context("failed to read metadata")?.len() == 0;
+
+    let mut records: IndexRecords = if !is_empty {
+        let file = BufReader::new(&file);
+        serde_json::from_reader(file).context("failed to deserialize file contents")?
+    } else {
+        IndexRecords::new()
+    };
+
+    func(&mut records);
+
+    {
+        file.seek(SeekFrom::Start(0))
+            .with_context(|| "failed to seek file cursor".to_string())?;
+        file.set_len(0)
+            .with_context(|| "failed to truncate file".to_string())?;
+
+        let file = BufWriter::new(file);
+        serde_json::to_writer(file, &records).context("failed to serialize file")?;
+    }
+
+    Ok(())
+}

scarb/src/core/registry/client/mod.rs

@@ -5,7 +5,8 @@ use anyhow::Result;
 use async_trait::async_trait;
 
 use crate::core::registry::index::IndexRecords;
-use crate::core::{PackageId, PackageName};
+use crate::core::{Package, PackageId, PackageName};
+use crate::flock::FileLockGuard;
 
 pub mod local;
 
@@ -43,4 +44,26 @@ pub trait RegistryClient: Send + Sync {
     /// it should write downloaded files to Scarb cache directory. If the file has already been
     /// downloaded, it should avoid downloading it again, and read it from this cache instead.
     async fn download(&self, package: PackageId) -> Result<PathBuf>;
+
+    /// State whether packages can be published to this registry.
+    ///
+    /// This method is permitted to do network lookups, for example to fetch registry config.
+    async fn supports_publish(&self) -> Result<bool> {
+        Ok(false)
+    }
+
+    /// Publish a package to this registry.
+    ///
+    /// This function can only be called if [`RegistryClient::supports_publish`] returns `true`.
+    /// Default implementation panics with [`unreachable!`].
+    ///
+    /// The `package` argument must correspond to just packaged `tarball` file.
+    /// The client is free to use information within `package` to send to the registry.
+    /// Package source is not required to match the registry the package is published to.
+    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<()> {
+        // Silence clippy warnings without using _ in argument names.
+        let _ = package;
+        let _ = tarball;
+        unreachable!("This registry does not support publishing.")
+    }
 }

scarb/src/internal/fsx.rs

@@ -100,6 +100,16 @@ pub fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<()> {
     }
 }
 
+/// Equivalent to [`fs::copy`] with better error messages.
+pub fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<u64> {
+    return inner(from.as_ref(), to.as_ref());
+
+    fn inner(from: &Path, to: &Path) -> Result<u64> {
+        fs::copy(from, to)
+            .with_context(|| format!("failed to copy file {} to {}", from.display(), to.display()))
+    }
+}
+
 pub trait PathUtf8Ext {
     fn try_as_utf8(&'_ self) -> Result<&'_ Utf8Path>;
 

scarb/src/ops/mod.rs

@@ -10,6 +10,7 @@ pub use manifest::*;
 pub use metadata::*;
 pub use new::*;
 pub use package::*;
+pub use publish::*;
 pub use resolve::*;
 pub use scripts::*;
 pub use subcommands::*;
@@ -24,6 +25,7 @@ mod manifest;
 mod metadata;
 mod new;
 mod package;
+mod publish;
 mod resolve;
 mod scripts;
 mod subcommands;

scarb/src/ops/package.rs

@@ -63,6 +63,14 @@ pub fn package(
         .collect()
 }
 
+pub fn package_one(
+    package_id: PackageId,
+    opts: &PackageOpts,
+    ws: &Workspace<'_>,
+) -> Result<FileLockGuard> {
+    package(&[package_id], opts, ws).map(|mut v| v.pop().unwrap())
+}
+
 #[tracing::instrument(level = "debug", skip(opts, ws))]
 pub fn package_list(
     packages: &[PackageId],