Merge pull request #404 from zmrow/clone-subcommand

tuftool: Add `clone` subcommand

tuftool/src/clone.rs

@@ -0,0 +1,147 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+use crate::common::UNUSED_URL;
+use crate::download_root::download_root;
+use crate::error::{self, Result};
+use snafu::ResultExt;
+use std::fs::File;
+use std::num::NonZeroU64;
+use std::path::PathBuf;
+use structopt::StructOpt;
+use tough::{ExpirationEnforcement, RepositoryLoader};
+use url::Url;
+
+#[derive(Debug, StructOpt)]
+pub(crate) struct CloneArgs {
+    /// Path to root.json file for the repository
+    #[structopt(
+        short = "r",
+        long = "root",
+        required_if("allow-root-download", "false")
+    )]
+    root: Option<PathBuf>,
+
+    /// Remote root.json version number
+    #[structopt(short = "v", long = "root-version", default_value = "1")]
+    root_version: NonZeroU64,
+
+    /// TUF repository metadata base URL
+    #[structopt(short = "m", long = "metadata-url")]
+    metadata_base_url: Url,
+
+    /// TUF repository targets base URL
+    #[structopt(short = "t", long = "targets-url", required_unless = "metadata-only")]
+    targets_base_url: Option<Url>,
+
+    /// Allow downloading the root.json file (unsafe)
+    #[structopt(long)]
+    allow_root_download: bool,
+
+    /// Allow repo download for expired metadata (unsafe)
+    #[structopt(long)]
+    allow_expired_repo: bool,
+
+    /// Download only these targets, if specified
+    #[structopt(short = "n", long = "target-names", conflicts_with = "metadata-only")]
+    target_names: Vec<String>,
+
+    /// Output directory of targets
+    #[structopt(long, required_unless = "metadata-only")]
+    targets_dir: Option<PathBuf>,
+
+    /// Output directory of metadata
+    #[structopt(long)]
+    metadata_dir: PathBuf,
+
+    /// Only download the repository metadata, not the targets
+    #[structopt(long, conflicts_with_all(&["target-names", "targets-dir", "targets-base-url"]))]
+    metadata_only: bool,
+}
+
+#[rustfmt::skip]
+fn expired_repo_warning() {
+    eprintln!("\
+=================================================================
+WARNING: repo metadata is expired, meaning the owner hasn't verified its contents lately and it could be unsafe!
+=================================================================");
+}
+
+impl CloneArgs {
+    pub(crate) fn run(&self) -> Result<()> {
+        // Use local root.json or download from repository
+        let root_path = if let Some(path) = &self.root {
+            PathBuf::from(path)
+        } else if self.allow_root_download {
+            let outdir = std::env::current_dir().context(error::CurrentDir)?;
+            download_root(&self.metadata_base_url, self.root_version, outdir)?
+        } else {
+            eprintln!("No root.json available");
+            std::process::exit(1);
+        };
+
+        // Structopt won't allow `targets_base_url` to be None when it is required.  We require the
+        // user to supply `targets_base_url` in the case they actually plan to download targets.
+        // When downloading metadata, we don't ever need to access the targets URL, so we use a
+        // fake URL to satisfy the library.
+        let targets_base_url = self
+            .targets_base_url
+            .as_ref()
+            .unwrap_or(&Url::parse(UNUSED_URL).context(error::UrlParse {
+                url: UNUSED_URL.to_owned(),
+            })?)
+            .clone();
+
+        // Load repository
+        let expiration_enforcement = if self.allow_expired_repo {
+            expired_repo_warning();
+            ExpirationEnforcement::Unsafe
+        } else {
+            ExpirationEnforcement::Safe
+        };
+        let repository = RepositoryLoader::new(
+            File::open(&root_path).context(error::OpenRoot { path: &root_path })?,
+            self.metadata_base_url.clone(),
+            targets_base_url,
+        )
+        .expiration_enforcement(expiration_enforcement)
+        .load()
+        .context(error::RepoLoad)?;
+
+        // Clone the repository, downloading none, all, or a subset of targets
+        if self.metadata_only {
+            println!("Cloning repository metadata to {:?}", self.metadata_dir);
+            repository
+                .cache_metadata(&self.metadata_dir, true)
+                .context(error::CloneRepository)?;
+        } else {
+            // Similar to `targets_base_url, structopt's guard rails won't let us have a
+            // `targets_dir` that is None when the argument is required.  We only require the user
+            // to supply a targets directory if they actually plan on downloading targets.
+            let targets_dir = self.targets_dir.as_ref().expect(
+                "Developer error: `targets_dir` is required unless downloading metadata only",
+            );
+
+            println!(
+                "Cloning repository:\n\tmetadata location: {:?}\n\ttargets location: {:?}",
+                self.metadata_dir, targets_dir
+            );
+            if self.target_names.is_empty() {
+                repository
+                    .cache(&self.metadata_dir, &targets_dir, None::<&[&str]>, true)
+                    .context(error::CloneRepository)?;
+            } else {
+                repository
+                    .cache(
+                        &self.metadata_dir,
+                        &targets_dir,
+                        Some(self.target_names.as_slice()),
+                        true,
+                    )
+                    .context(error::CloneRepository)?;
+            }
+        };
+
+        Ok(())
+    }
+}

tuftool/src/download.rs

@@ -1,10 +1,11 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
+use crate::download_root::download_root;
 use crate::error::{self, Result};
 use snafu::{OptionExt, ResultExt};
 use std::fs::File;
-use std::io::{self};
+use std::io;
 use std::num::NonZeroU64;
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
@@ -18,8 +19,8 @@ pub(crate) struct DownloadArgs {
     root: Option<PathBuf>,
 
     /// Remote root.json version number
-    #[structopt(short = "v", long = "root-version")]
-    root_version: Option<NonZeroU64>,
+    #[structopt(short = "v", long = "root-version", default_value = "1")]
+    root_version: NonZeroU64,
 
     /// TUF repository metadata base URL
     #[structopt(short = "m", long = "metadata-url")]
@@ -45,16 +46,6 @@ pub(crate) struct DownloadArgs {
     allow_expired_repo: bool,
 }
 
-fn root_warning<P: AsRef<Path>>(path: P) {
-    #[rustfmt::skip]
-    eprintln!("\
-=================================================================
-WARNING: Downloading root.json to {}
-This is unsafe and will not establish trust, use only for testing
-=================================================================",
-              path.as_ref().display());
-}
-
 fn expired_repo_warning<P: AsRef<Path>>(path: P) {
     #[rustfmt::skip]
     eprintln!("\
@@ -71,28 +62,8 @@ impl DownloadArgs {
         let root_path = if let Some(path) = &self.root {
             PathBuf::from(path)
         } else if self.allow_root_download {
-            let name = if let Some(version) = self.root_version {
-                format!("{}.root.json", version)
-            } else {
-                String::from("1.root.json")
-            };
-            let path = std::env::current_dir()
-                .context(error::CurrentDir)?
-                .join(&name);
-            let url = self
-                .metadata_base_url
-                .join(&name)
-                .context(error::UrlParse {
-                    url: self.metadata_base_url.as_str(),
-                })?;
-            root_warning(&path);
-
-            let mut f = File::create(&path).context(error::OpenFile { path: &path })?;
-            reqwest::blocking::get(url.as_str())
-                .context(error::ReqwestGet)?
-                .copy_to(&mut f)
-                .context(error::ReqwestCopy)?;
-            path
+            let outdir = std::env::current_dir().context(error::CurrentDir)?;
+            download_root(&self.metadata_base_url, self.root_version, outdir)?
         } else {
             eprintln!("No root.json available");
             std::process::exit(1);

tuftool/src/download_root.rs

@@ -0,0 +1,51 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//! The `download_root` module owns the logic for downloading a given version of `root.json`.
+
+use crate::error::{self, Result};
+use snafu::ResultExt;
+use std::fs::File;
+use std::num::NonZeroU64;
+use std::path::{Path, PathBuf};
+use url::Url;
+
+/// Download the given version of `root.json`
+/// This is an unsafe operation, and doesn't establish trust. It should only be used for testing!
+pub(crate) fn download_root<P>(
+    metadata_base_url: &Url,
+    version: NonZeroU64,
+    outdir: P,
+) -> Result<PathBuf>
+where
+    P: AsRef<Path>,
+{
+    let name = format!("{}.root.json", version);
+
+    let path = outdir.as_ref().join(&name);
+    let url = metadata_base_url.join(&name).context(error::UrlParse {
+        url: format!("{}/{}", metadata_base_url.as_str(), name),
+    })?;
+    root_warning(&path);
+
+    let mut root_request = reqwest::blocking::get(url.as_str())
+        .context(error::ReqwestGet)?
+        .error_for_status()
+        .context(error::BadResponse { url })?;
+
+    let mut f = File::create(&path).context(error::OpenFile { path: &path })?;
+    root_request.copy_to(&mut f).context(error::ReqwestCopy)?;
+
+    Ok(path)
+}
+
+/// Print a very noticeable warning message about the unsafe nature of downloading `root.json`
+/// without verification
+fn root_warning<P: AsRef<Path>>(path: P) {
+    #[rustfmt::skip]
+    eprintln!("\
+=================================================================
+WARNING: Downloading root.json to {}
+This is unsafe and will not establish trust, use only for testing
+=================================================================",
+              path.as_ref().display());
+}

tuftool/src/error.rs

@@ -13,6 +13,12 @@ pub(crate) type Result<T> = std::result::Result<T, Error>;
 #[derive(Debug, Snafu)]
 #[snafu(visibility = "pub(crate)")]
 pub(crate) enum Error {
+    #[snafu(display("Failed to clone repository: {}", source))]
+    CloneRepository {
+        source: tough::error::Error,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to run {}: {}", command_str, source))]
     CommandExec {
         command_str: String,
@@ -243,6 +249,13 @@ pub(crate) enum Error {
         backtrace: Backtrace,
     },
 
+    #[snafu(display("Response '{}' from '{}': {}", get_status_code(source), url, source))]
+    BadResponse {
+        url: String,
+        source: reqwest::Error,
+        backtrace: Backtrace,
+    },
+
     #[snafu(display("Failed to sign repository: {}", source))]
     SignRepo {
         source: tough::error::Error,
@@ -357,3 +370,12 @@ pub(crate) enum Error {
         backtrace: Backtrace,
     },
 }
+
+// Extracts the status code from a reqwest::Error and converts it to a string to be displayed
+fn get_status_code(source: &reqwest::Error) -> String {
+    source
+        .status()
+        .as_ref()
+        .map_or("Unknown", |i| i.as_str())
+        .to_string()
+}

tuftool/src/main.rs

@@ -13,11 +13,13 @@
 
 mod add_key_role;
 mod add_role;
+mod clone;
 mod common;
 mod create;
 mod create_role;
 mod datetime;
 mod download;
+mod download_root;
 mod error;
 mod remove_key_role;
 mod remove_role;
@@ -76,14 +78,16 @@ impl Program {
 enum Command {
     /// Create a TUF repository
     Create(create::CreateArgs),
-    /// Download a TUF repository's resources
+    /// Download a TUF repository's targets
     Download(download::DownloadArgs),
     /// Update a TUF repository's metadata and optionally add targets
     Update(Box<update::UpdateArgs>),
     /// Manipulate a root.json metadata file
     Root(root::Command),
     /// Delegation Commands
     Delegation(Delegation),
+    /// Clone a TUF repository, including metadata and some or all targets
+    Clone(clone::CloneArgs),
 }
 
 impl Command {
@@ -94,6 +98,7 @@ impl Command {
             Command::Download(args) => args.run(),
             Command::Update(args) => args.run(),
             Command::Delegation(cmd) => cmd.run(),
+            Command::Clone(cmd) => cmd.run(),
         }
     }
 }