From dd46205e242a6b876da1318c62e8fe072ce8494b Mon Sep 17 00:00:00 2001 From: xphoniex Date: Mon, 14 Feb 2022 12:37:04 +0000 Subject: [PATCH] rad-ci: Initial commit Signed-off-by: xphoniex --- Cargo.lock | 36 +++++ Cargo.toml | 5 +- ci/Cargo.toml | 16 ++ ci/extract-env | 15 ++ ci/post-receive-ok | 102 ++++++++++++ ci/src/lib.rs | 394 +++++++++++++++++++++++++++++++++++++++++++++ src/bin/rad-ci.rs | 5 + 7 files changed, 572 insertions(+), 1 deletion(-) create mode 100644 ci/Cargo.toml create mode 100755 ci/extract-env create mode 100755 ci/post-receive-ok create mode 100644 ci/src/lib.rs create mode 100644 src/bin/rad-ci.rs diff --git a/Cargo.lock b/Cargo.lock index e5a53cac..3aef9c64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4735,6 +4735,19 @@ dependencies = [ "rad-terminal", ] +[[package]] +name = "rad-ci" +version = "0.1.0" +dependencies = [ + "anyhow", + "lexopt", + "rad-common", + "rad-terminal", + "ssh2", + "url", + "whoami", +] + [[package]] name = "rad-clib" version = "0.1.0" @@ -5112,6 +5125,7 @@ dependencies = [ "rad-account", "rad-auth", "rad-checkout", + "rad-ci", "rad-clone", "rad-common", "rad-ens", @@ -5968,6 +5982,18 @@ dependencies = [ "der 0.5.1", ] +[[package]] +name = "ssh2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269343e64430067a14937ae0e3c4ec604c178fb896dde0964b1acd22b3e2eeb1" +dependencies = [ + "bitflags", + "libc", + "libssh2-sys", + "parking_lot", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -6684,6 +6710,16 @@ dependencies = [ "cc", ] +[[package]] +name = "whoami" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524b58fa5a20a2fb3014dd6358b70e6579692a56ef6fce928834e488f42f65e8" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index 3d5ebcdb..5c3e036d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ assets = [ ["target/release/rad-checkout", "usr/bin/rad-checkout", "755"], ["target/release/rad-untrack", "usr/bin/rad-untrack", "755"], ["target/release/git-remote-rad", "usr/bin/git-remote-rad", "755"], + ["target/release/rad-ci", "usr/bin/rad-ci", "755"], ["radicle-tools.1.gz", "usr/share/man/man1/radicle-tools.1.gz", "644"], ] @@ -48,6 +49,7 @@ rad-untrack = { path = "./untrack" } rad-help = { path = "./help" } rad-ls = { path = "./ls" } rad-rm = { path = "./rm" } +rad-ci = { path = "./ci" } ethers = { version = "0.6.2" } link-identities = { version = "0" } radicle-git-helpers = { version = "0" } @@ -76,7 +78,8 @@ members = [ "track", "untrack", "proof-generator", - "authorized-keys" + "authorized-keys", + "ci" ] [patch.crates-io.link-crypto] diff --git a/ci/Cargo.toml b/ci/Cargo.toml new file mode 100644 index 00000000..0b8fbb2b --- /dev/null +++ b/ci/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rad-ci" +version = "0.1.0" +authors = ["The Radicle Team "] +edition = "2018" +license = "MIT OR Apache-2.0" +description = "Install or uninstall ci on your seed node" + +[dependencies] +anyhow = "1.0" +lexopt = { version = "0.2" } +rad-common = { path = "../common" } +rad-terminal = { path = "../terminal" } +url = { version = "*" } +ssh2 = "0.9.3" +whoami = "1.2.1" diff --git a/ci/extract-env b/ci/extract-env new file mode 100755 index 00000000..0deb183c --- /dev/null +++ b/ci/extract-env @@ -0,0 +1,15 @@ +#!/bin/sh + +set -ue + +file=$1 +port=$(cat $file | grep ports | cut -d \" -f2 | cut -d \: -f1) +user=$(cat $file | grep "CONCOURSE_ADD_LOCAL_USER" | cut -d \: -f2 | cut -d ' ' -f2) +pass=$(cat $file | grep "CONCOURSE_ADD_LOCAL_USER" | cut -d \: -f3) + +cat < $2 +CONCOURSE_USER=$user +CONCOURSE_PASS=$pass +CONCOURSE_URL=http://localhost:$port +END + diff --git a/ci/post-receive-ok b/ci/post-receive-ok new file mode 100755 index 00000000..d8433d1f --- /dev/null +++ b/ci/post-receive-ok @@ -0,0 +1,102 @@ +#!/bin/sh + +echo "**** ***** Running Radicle CI Hook ***** ****" + +if [ -f "$GIT_PROJECT_ROOT/etc/.ci.env" ]; then + . "$GIT_PROJECT_ROOT/etc/.ci.env" +fi + +run_pipeline() { + local tag=$(echo $GIT_NAMESPACE | cut -c 30-) + local urn="$RADICLE_NAME-$tag" + # login + fly -t local login -c $CONCOURSE_URL -u $CONCOURSE_USER -p $CONCOURSE_PASS + # create pipeline + if [ -f "$GIT_PROJECT_ROOT/secrets/$GIT_NAMESPACE.yml" ]; then + fly -t local validate-pipeline -c /tmp/$GIT_NAMESPACE/.rad/concourse.yml + fly -t local set-pipeline -p $urn -c /tmp/$GIT_NAMESPACE/.rad/concourse.yml --non-interactive -l $GIT_PROJECT_ROOT/secrets/$GIT_NAMESPACE.yml > /dev/null + else + fly -t local set-pipeline -p $urn -c /tmp/$GIT_NAMESPACE/.rad/concourse.yml --non-interactive + fi + # unpause pipeline + fly -t local unpause-pipeline -p $urn + # trigger a job + # TODO: trigger all jobs + fly -t local trigger-job --job $urn/job +} + +delete_team() { + local tag=$(echo $GIT_NAMESPACE | cut -c 31-) + local urn="$RADICLE_NAME-$tag" + # login + fly -t local login -c $CONCOURSE_URL -u $CONCOURSE_USER -p $CONCOURSE_PASS + # destroy pipeline + fly -t local destroy-pipeline -p $urn -n +} + +check_ci_deletion() { + local urn=$1 + local commit=$2 + local branch=$3 + # initial commit has prev_commit as 00..00 + if [ "$commit" = "0000000000000000000000000000000000000000" ]; then + return; + fi + + # checkout into that commit + unset GIT_DIR && cd /tmp/$urn && git checkout $commit -f -q + + if [ -f "/tmp/$urn/.rad/concourse.yml" ]; then + echo "ci yml was deleted in this commit" + delete_team $urn + fi + + # revert checkout + unset GIT_DIR && cd /tmp/$urn && git checkout $branch -f +} + +clone_or_pull() { + local urn=$GIT_NAMESPACE + if [ -e "/tmp/$urn" ]; then + echo "pulling..." + unset GIT_DIR && cd /tmp/$urn && git pull -f --rebase + else + echo "cloning..." + git clone $GIT_DIR /tmp/$urn + fi +} + +while read line +do + echo "stdin: $line" + urn=$GIT_NAMESPACE + prev_commit=$(echo $line | cut -d' ' -f2) + branch=$(echo $line | cut -d' ' -f4) + echo "urn: $urn, project: $RADICLE_NAME, prev_commit: $prev_commit, branch: $branch" + + # clone the project locally in /tmp + clone_or_pull + + default_branch=$(GIT_DIR=/tmp/$urn/.git git symbolic-ref --short refs/remotes/origin/HEAD) + # remove the `origin` part + default_branch=$(echo $default_branch | cut -c 8-) + + if [ "$branch" != "$default_branch" ]; then + echo "skpping, only running on pushes to branch $default_branch" + continue + fi + + # check for .rad/concourse.yml + if [ -f "/tmp/$urn/.rad/concourse.yml" ]; then + echo "yml exists." + run_pipeline + else + echo "yml does not exist." + check_ci_deletion $urn $prev_commit $branch + exit 0 + fi + +done + +echo "Exiting..." +exit 0 diff --git a/ci/src/lib.rs b/ci/src/lib.rs new file mode 100644 index 00000000..24be6353 --- /dev/null +++ b/ci/src/lib.rs @@ -0,0 +1,394 @@ +use std::convert::TryInto; +use std::ffi::OsString; +use std::fs::File; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; + +use anyhow::Context; +use ssh2::Session; +use url::Host; + +use rad_terminal::args::{Args, Help}; +use rad_terminal::components as term; + +pub const HELP: Help = Help { + name: "ci", + description: env!("CARGO_PKG_DESCRIPTION"), + version: env!("CARGO_PKG_VERSION"), + usage: r#" +USAGE + rad ci (--install | --uninstall) --seed [--ssh-user ] [--root ] [--config ] [--verbose] + +OPTIONS + --install Runs docker images for concourse + --uninstall Terminates running concourse images + --seed Address of the seed node + --ssh-user SSH user on target machine. Default: root + --root (Optional) Radicle root on server. Default; /app/radicle/radicle + --config (Optional) docker-compose.yml to deploy. Default: downloaded + --verbose +"#, +}; + +const CONCOURSE_DOCKER_COMPOSE_URL: &str = "https://concourse-ci.org/docker-compose.yml"; +const YML_NAME: &str = "concourse-docker-compose.yml"; + +#[derive(Default, Eq, PartialEq)] +pub struct Options { + pub install: bool, + pub uninstall: bool, + pub seed: Option, + pub ssh_user: String, + pub root: PathBuf, + pub config: Option, + pub verbose: bool, +} + +impl Args for Options { + fn from_args(args: Vec) -> anyhow::Result<(Self, Vec)> { + use lexopt::prelude::*; + + let mut parser = lexopt::Parser::from_args(args); + let mut install = false; + let mut uninstall = false; + let mut seed = None; + let mut ssh_user = whoami::username(); + let mut root = PathBuf::from("/app/radicle/root"); + let mut config = None; + let mut verbose = false; + + while let Some(arg) = parser.next()? { + match arg { + Long("seed") => { + let value = parser.value()?; + let value = value.to_string_lossy(); + let value = value.as_ref(); + let addr = Host::parse(value)?; + + seed = Some(addr); + } + Long("install") | Short('i') => { + install = true; + } + Long("uninstall") | Short('u') => { + uninstall = true; + } + Long("ssh-user") => { + let value = parser.value()?; + + ssh_user = String::from(value.to_string_lossy()); + } + Long("root") => { + let value = parser.value()?; + root = value.try_into()?; + } + Long("config") => { + let value = parser.value()?; + let path: PathBuf = value.try_into()?; + + config = Some(path); + } + Long("verbose") | Short('v') => { + verbose = true; + } + _ => { + return Err(anyhow::anyhow!(arg.unexpected())); + } + } + } + + Ok(( + Options { + seed, + install, + uninstall, + ssh_user, + root, + config, + verbose, + }, + vec![], + )) + } +} + +struct ServerPath { + root: PathBuf, +} + +impl ServerPath { + fn etc(&self) -> PathBuf { + self.root.join("etc") + } + + fn docker_compose(&self) -> PathBuf { + self.etc().join(YML_NAME) + } + + fn hook(&self) -> PathBuf { + self.root.join("git").join("hooks").join("post-receive-ok") + } + + fn extract_env(&self) -> PathBuf { + self.etc().join("extract-env") + } + + fn ci_env(&self) -> PathBuf { + self.etc().join(".ci.env") + } +} + +pub fn run(options: Options) -> anyhow::Result<()> { + let seed = options.seed.context("Seed is invalid")?; + // SSH into server + let spinner = term::spinner(&format!( + "SSH into remote {}@{}", + term::format::bold(&options.ssh_user), + term::format::dim(&seed) + )); + let tcp = TcpStream::connect(format!("{}:22", &seed))?; + let mut sess = Session::new()?; + sess.set_tcp_stream(tcp); + sess.handshake()?; + + let mut agent = sess.agent()?; + agent.connect()?; + agent.list_identities()?; + for identity in agent.identities()? { + if agent.userauth(&options.ssh_user, &identity).is_ok() && sess.authenticated() { + break; + } + } + if !sess.authenticated() { + anyhow::bail!("Couldn't authenticate against server, add your key to ssh-agent."); + } + spinner.finish(); + + let verbosity = options.verbose; + let execute_fn = |cmd, spinner| execute_cmd_with_spinner(&sess, cmd, spinner, verbosity); + + // Check requirements + term::blank(); + term::info!("Checking requirements:"); + term::blank(); + + let mut requirements = vec![ + "wget", + "sh", + "cat", + "grep", + "cut", + "mkdir", + "docker", + "docker-compose", + ]; + if options.config.is_some() { + requirements.remove(0); + } + + if requirements + .iter() + .map(|req| { + let spinner = term::spinner(&format!( + "Checking if {} exists", + term::format::highlight(req) + )); + let (_, status) = execute_fn(format!("which {}", req), spinner).unwrap(); + status + }) + .sum::() + > 0 + { + term::info!("Requirements are not installed."); + std::process::exit(1); + } + term::blank(); + + let server_path = ServerPath { root: options.root }; + + // Apply changes + if options.install { + // Make sure directories exist + let spinner = term::spinner(&format!( + "Making sure directory {:?} exists", + server_path.etc() + )); + let (_, status) = execute_fn(format!("mkdir -p {:?}", server_path.etc()), spinner)?; + if status != 0 { + std::process::exit(1); + } + term::blank(); + + // Upload custom config file (docker-compose.yml) when supplied + if options.config.is_some() { + let spinner = term::spinner(&format!( + "Copying config yaml to {:?}", + server_path.docker_compose() + )); + let path = options.config.unwrap(); + let mut file = File::open(path).context("Couldn't open config file")?; + let mut content = String::new(); + file.read_to_string(&mut content) + .context("Couldn't read the contents of config file")?; + + upload_content_to_file( + &sess, + content.as_bytes(), + server_path.docker_compose().to_str().unwrap(), + 0o644, + )?; + spinner.finish(); + } + + // Copy post-receive-ok file over + let spinner = term::spinner(&format!( + "Copying post-receive-ok hook to {:?}", + server_path.hook() + )); + let receive_hook_content = include_str!("../post-receive-ok"); + upload_content_to_file( + &sess, + receive_hook_content.as_bytes(), + server_path + .hook() + .to_str() + .context("Couldn't get server hook path as str")?, + 0o755, + )?; + spinner.finish(); + + // Download the docker-compose.yml + let spinner = term::spinner(&format!( + "Downloading docker-compose.yml to {:?}", + server_path.docker_compose() + )); + execute_fn( + format!( + "wget -nc -O {:?} {}", + server_path.docker_compose(), + CONCOURSE_DOCKER_COMPOSE_URL + ), + spinner, + )?; + + // Create .ci.env + let spinner = term::spinner(&format!( + "Copying extract-env to {:?}", + server_path.extract_env() + )); + let extract_env_content = include_str!("../extract-env"); + upload_content_to_file( + &sess, + extract_env_content.as_bytes(), + server_path + .extract_env() + .to_str() + .context("Couldn't get extract-env path as str")?, + 0o755, + )?; + spinner.finish(); + + let spinner = term::spinner(&format!("Creating .ci.env in {:?}", server_path.ci_env())); + let (_, status) = execute_fn( + format!( + "{:?} {:?} {:?} && cat {:?}", + server_path + .extract_env() + .to_str() + .context("Couldn't get extract-env path as str")?, + server_path.docker_compose(), + server_path.ci_env(), + server_path.ci_env() + ), + spinner, + )?; + if status != 0 { + std::process::exit(1); + } + + // Run `docker-compose up -d` + term::blank(); + let spinner = term::spinner("Installing Concourse CI..."); + let (_, status) = execute_fn( + format!("docker-compose -f {:?} up -d", server_path.docker_compose()), + spinner, + )?; + if status == 0 { + term::success!("Concourse CI installed!"); + } + } else if options.uninstall { + // Run `docker-compose down` + let spinner = term::spinner("Uninstalling Concourse CI..."); + let (_, status) = execute_fn( + format!("docker-compose -f {:?} down", server_path.docker_compose()), + spinner, + )?; + if status == 0 { + term::success!("Concourse CI uninstalled!"); + } + + // Delete docker-compose.yml file + let mut channel = sess.channel_session().unwrap(); + execute_cmd( + &mut channel, + &format!("rm {:?}", server_path.docker_compose()), + )?; + + // Delete post-receive-ok hook + let mut channel = sess.channel_session().unwrap(); + execute_cmd(&mut channel, &format!("rm {:?}", server_path.hook()))?; + } + + Ok(()) +} + +fn execute_cmd(channel: &mut ssh2::Channel, cmd: &str) -> anyhow::Result<(String, i32)> { + let mut res = String::new(); + channel.exec(cmd)?; + channel.read_to_string(&mut res)?; + channel.stderr().read_to_string(&mut res)?; + channel.wait_close()?; + Ok((res, channel.exit_status()?)) +} + +fn execute_cmd_with_spinner( + sess: &ssh2::Session, + cmd: String, + spinner: term::Spinner, + verbose: bool, +) -> anyhow::Result<(String, i32)> { + let mut channel = sess.channel_session().unwrap(); + let (res, status) = execute_cmd(&mut channel, &cmd)?; + if status == 0 { + spinner.finish(); + } else { + spinner.failed(); + } + if verbose { + term::blob(&res); + } + Ok((res, status)) +} + +fn upload_content_to_file( + sess: &ssh2::Session, + content: &[u8], + file_path: &str, + permissions: i32, +) -> anyhow::Result<()> { + let mut remote_file = sess.scp_send( + Path::new(file_path), + permissions, + content.len().try_into().unwrap(), + None, + )?; + + remote_file.write_all(content)?; + remote_file.send_eof()?; + remote_file.wait_eof()?; + remote_file.close()?; + remote_file.wait_close()?; + + Ok(()) +} diff --git a/src/bin/rad-ci.rs b/src/bin/rad-ci.rs new file mode 100644 index 00000000..f848aa6d --- /dev/null +++ b/src/bin/rad-ci.rs @@ -0,0 +1,5 @@ +use rad_ci::{run, Options, HELP}; + +fn main() { + rad_terminal::args::run_command::(HELP, "ci", run); +}