Implement base data models for registry index (#767)

**Stack**:
- #794
- #793
- #790
- #792
- #783
- #767 ⬅


⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do
not merge manually using the UI - doing so may have unexpected results.*

scarb/src/core/registry/index/base_url.rs

@@ -0,0 +1,67 @@
+use std::ops::Deref;
+use std::str::FromStr;
+
+use anyhow::ensure;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+/// Wrapper over [`Url`] which ensures that the URL can be a joined, and has trailing slash.
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(try_from = "Url")]
+pub struct BaseUrl(Url);
+
+impl Deref for BaseUrl {
+    type Target = Url;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl TryFrom<Url> for BaseUrl {
+    type Error = anyhow::Error;
+
+    fn try_from(mut url: Url) -> Result<Self, Self::Error> {
+        ensure!(!url.cannot_be_a_base(), "invalid base url: {url}");
+
+        if !url.path().ends_with('/') {
+            url.set_path(&format!("{}/", url.path()));
+        }
+
+        Ok(Self(url))
+    }
+}
+
+impl FromStr for BaseUrl {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        BaseUrl::try_from(Url::parse(s)?)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+    use test_case::test_case;
+
+    use super::BaseUrl;
+
+    #[test]
+    fn rejects_cannot_be_a_base_urls() {
+        assert_eq!(
+            "invalid base url: data:text/plain,Stuff",
+            BaseUrl::from_str("data:text/plain,Stuff")
+                .unwrap_err()
+                .to_string(),
+        );
+    }
+
+    #[test_case("https://example.com" => "https://example.com/")]
+    #[test_case("https://example.com/file" => "https://example.com/file/")]
+    #[test_case("https://example.com/path/" => "https://example.com/path/")]
+    #[test_case("https://example.com/file.json" => "https://example.com/file.json/")]
+    fn appends_trailing_slash_if_missing(url: &str) -> String {
+        BaseUrl::from_str(url).unwrap().to_string()
+    }
+}

scarb/src/core/registry/index/config.rs

@@ -0,0 +1,92 @@
+use anyhow::{ensure, Result};
+use serde::{Deserialize, Serialize};
+
+use crate::core::registry::index::{BaseUrl, TemplateUrl};
+
+/// The `config.json` file stored in and defining the index.
+///
+/// The config file may look like this:
+///
+/// ```json
+/// {
+///   "version": 1,
+///   "api": "https://example.com/api/v1",
+///   "dl": "https://example.com/api/v1/download/{package}/{version}",
+///   "index": "https://example.com/index/{prefix}/{package}.json"
+/// }
+/// ```
+///
+/// ## URL Templates
+///
+/// The values for the `"dl"` and `"index"` fields are URL templates.
+/// See documentation for [`TemplateUrl`] for supported expansion patterns.
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct IndexConfig {
+    /// Index version, must be `1` (numeric).
+    pub version: IndexVersion,
+
+    /// API endpoint for the registry.
+    ///
+    /// This is what's actually hit to perform operations like yanks, owner modifications,
+    /// publish new packages, etc.
+    /// If this is `None`, the registry does not support API commands.
+    pub api: Option<BaseUrl>,
+
+    /// Download endpoint for all packages.
+    pub dl: TemplateUrl,
+
+    /// Base URL for main index file files.
+    ///
+    /// Usually, this is a location where `config.json` lies, as the rest of index files resides
+    /// alongside config.
+    pub index: TemplateUrl,
+}
+
+#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(into = "u8", try_from = "u8")]
+pub struct IndexVersion;
+
+impl TryFrom<u8> for IndexVersion {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        ensure!(value == 1, "unsupported index version: {value}");
+        Ok(Self)
+    }
+}
+
+impl From<IndexVersion> for u8 {
+    fn from(_: IndexVersion) -> Self {
+        1
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::core::registry::index::TemplateUrl;
+
+    use super::IndexConfig;
+
+    #[test]
+    fn deserialize() {
+        let expected = IndexConfig {
+            version: Default::default(),
+            api: Some("https://example.com/api/v1/".parse().unwrap()),
+            dl: TemplateUrl::new("https://example.com/api/v1/download/{package}/{version}"),
+            index: TemplateUrl::new("https://example.com/index/{prefix}/{package}.json"),
+        };
+
+        let actual: IndexConfig = serde_json::from_str(
+            r#"{
+              "version": 1,
+              "api": "https://example.com/api/v1",
+              "dl": "https://example.com/api/v1/download/{package}/{version}",
+              "index": "https://example.com/index/{prefix}/{package}.json"
+            }"#,
+        )
+        .unwrap();
+
+        assert_eq!(actual, expected);
+    }
+}

scarb/src/core/registry/index/mod.rs

@@ -0,0 +1,12 @@
+// FIXME(mkaput): Remove this when we will indeed use this.
+#![allow(unused)]
+
+pub use base_url::*;
+pub use config::*;
+pub use record::*;
+pub use template_url::*;
+
+mod base_url;
+mod config;
+mod record;
+mod template_url;

scarb/src/core/registry/index/record.rs

@@ -0,0 +1,34 @@
+use semver::{Version, VersionReq};
+use serde::{Deserialize, Serialize};
+
+use crate::core::PackageName;
+
+pub type IndexRecords = Vec<IndexRecord>;
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub struct IndexRecord {
+    #[serde(rename = "v")]
+    pub version: Version,
+    #[serde(rename = "deps")]
+    pub dependencies: IndexDependencies,
+    #[serde(rename = "cksum")]
+    pub checksum: String,
+    #[serde(default = "default_false", skip_serializing_if = "is_false")]
+    pub no_core: bool,
+}
+
+pub type IndexDependencies = Vec<IndexDependency>;
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub struct IndexDependency {
+    pub name: PackageName,
+    pub req: VersionReq,
+}
+
+fn default_false() -> bool {
+    false
+}
+
+fn is_false(value: &bool) -> bool {
+    !*value
+}

scarb/src/core/registry/index/template_url.rs

@@ -0,0 +1,146 @@
+use std::fmt;
+
+use anyhow::{bail, Context, Result};
+use indoc::formatdoc;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+use crate::core::{PackageId, PackageName};
+
+/// A template string which will generate the download or index URL for the specific package.
+///
+/// The patterns `{package}` and `{version}` will be replaced with the package name and version
+/// (without leading `v`) respectively. The pattern `{prefix}` will be replaced with the package's
+/// prefix directory name (e.g. `ab/cd` for package named `abcd`).
+///
+/// If the template contains no patterns, the template will expand to itself, which probably is
+/// not something you want.
+///
+/// Upon expansion, the template must form a valid URL.
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct TemplateUrl(String);
+
+#[derive(Default)]
+pub struct ExpansionParams {
+    pub package: Option<String>,
+    pub version: Option<String>,
+}
+
+impl TemplateUrl {
+    pub fn new(template: &str) -> Self {
+        Self(template.to_owned())
+    }
+
+    pub fn expand(&self, params: ExpansionParams) -> Result<Url> {
+        let prefix = params.package.as_deref().map(pkg_prefix);
+
+        let replace = |s: &mut String, pattern: &str, expansion: Option<String>| -> Result<()> {
+            match expansion {
+                Some(expansion) => {
+                    *s = s.replace(pattern, &expansion);
+                    Ok(())
+                }
+                None if s.contains(pattern) => bail!(
+                    "pattern `{pattern}` in not available in this context for template url: {self}"
+                ),
+                None => Ok(()),
+            }
+        };
+
+        let mut expansion = self.0.clone();
+        replace(&mut expansion, "{package}", params.package)?;
+        replace(&mut expansion, "{version}", params.version)?;
+        replace(&mut expansion, "{prefix}", prefix)?;
+
+        expansion.parse().with_context(|| {
+            formatdoc! {r"
+                failed to expand template url:
+                template:  {self}
+                expansion: {expansion}
+            "}
+        })
+    }
+}
+
+impl fmt::Display for TemplateUrl {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}
+
+impl From<PackageName> for ExpansionParams {
+    fn from(package: PackageName) -> Self {
+        (&package).into()
+    }
+}
+
+impl From<&PackageName> for ExpansionParams {
+    fn from(package: &PackageName) -> Self {
+        Self {
+            package: Some(package.to_string()),
+            ..Default::default()
+        }
+    }
+}
+
+impl From<PackageId> for ExpansionParams {
+    fn from(package: PackageId) -> Self {
+        Self {
+            package: Some(package.name.to_string()),
+            version: Some(package.version.to_string()),
+        }
+    }
+}
+
+/// Make a path to a package directory, which aligns to the index directory layout.
+fn pkg_prefix(name: &str) -> String {
+    match name.len() {
+        1 => "1".to_string(),
+        2 => "2".to_string(),
+        3 => format!("3/{}", &name[..1]),
+        _ => format!("{}/{}", &name[0..2], &name[2..4]),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use test_case::test_case;
+
+    use crate::core::{PackageId, PackageName};
+
+    use super::TemplateUrl;
+
+    #[test]
+    fn expand() {
+        let template = TemplateUrl::new("https://example.com/{prefix}/{package}-{version}.json");
+        let package_id = PackageId::from_display_str("foobar v1.0.0").unwrap();
+        assert_eq!(
+            "https://example.com/fo/ob/foobar-1.0.0.json",
+            template.expand(package_id.into()).unwrap().as_str()
+        )
+    }
+
+    #[test]
+    fn expand_missing_pattern() {
+        let template = TemplateUrl::new("https://example.com/{prefix}/{package}-{version}.json");
+        assert_eq!(
+            template
+                .expand(PackageName::CORE.into())
+                .unwrap_err()
+                .to_string(),
+            "pattern `{version}` in not available in this context for template url: \
+            https://example.com/{prefix}/{package}-{version}.json"
+        );
+    }
+
+    #[test_case("a" => "1")]
+    #[test_case("ab" => "2")]
+    #[test_case("abc" => "3/a")]
+    #[test_case("Xyz" => "3/X")]
+    #[test_case("AbCd" => "Ab/Cd")]
+    #[test_case("pQrS" => "pQ/rS")]
+    fn pkg_prefix(input: &str) -> String {
+        super::pkg_prefix(input)
+    }
+}

scarb/src/core/registry/mod.rs

@@ -4,10 +4,13 @@ use async_trait::async_trait;
 use crate::core::{ManifestDependency, Package, PackageId, Summary};
 
 pub mod cache;
+pub mod index;
 pub mod patch_map;
 pub mod patcher;
 pub mod source_map;
 
+pub const DEFAULT_REGISTRY_INDEX: &str = "https://there-is-no-default-registry-yet.com";
+
 #[async_trait(?Send)]
 pub trait Registry {
     /// Attempt to find the packages that match a dependency request.

scarb/src/core/source/id.rs

@@ -9,6 +9,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use smol_str::SmolStr;
 use url::Url;
 
+use crate::core::registry::DEFAULT_REGISTRY_INDEX;
 use crate::core::source::Source;
 use crate::core::Config;
 use crate::internal::fsx::PathBufUtf8Ext;
@@ -152,7 +153,7 @@ impl SourceId {
 
     pub fn default_registry() -> Self {
         static CACHE: Lazy<SourceId> = Lazy::new(|| {
-            let url = Url::parse("https://there-is-no-default-registry-yet.com").unwrap();
+            let url = Url::parse(DEFAULT_REGISTRY_INDEX).unwrap();
             SourceId::new(url, SourceKind::Registry).unwrap()
         });
         *CACHE