tidy up some validation checks + add the actual command to the error output to retry

.circleci/config.yml

@@ -11,7 +11,7 @@ parameters:
 jobs:
   test:
     docker:
-      - image: cimg/rust:1.67
+      - image: cimg/rust:1.78
         environment:
           CARGO_NET_GIT_FETCH_WITH_CLI: true
       - image: postgres:13.4
@@ -41,7 +41,7 @@ jobs:
 
   build_release_linux_universal:
     docker:
-      - image: clux/muslrust:1.67.1
+      - image: clux/muslrust:1.78.0-stable
     environment:
       CARGO_NET_GIT_FETCH_WITH_CLI: true
     steps:
@@ -135,7 +135,7 @@ jobs:
       - run: brew install cmake
       - run: TAG="${CIRCLE_TAG:-v0.0.0}"; ./update_version $TAG
       - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-      - run: rustup install 1.67
+      - run: rustup install 1.78
       - run: rustup target add x86_64-apple-darwin
       - run: rustup target add aarch64-apple-darwin
       - run: cargo build --release --target=x86_64-apple-darwin

Cargo.toml

@@ -27,6 +27,7 @@ uuid = { version = "1.8", features = [ "v4"] }
 mimalloc = "0.1.41"
 log = "0.4.21"
 zstd = "0.13.1"
+colored = "2.1.0"
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"

build_and_test

@@ -1,8 +1,8 @@
 #!/bin/bash
 set -e
 
-cargo install cargo2junit --version 0.1.12
+cargo install cargo2junit --version 0.1.13
 cargo fmt -- --check
 cargo clippy --all-targets --all-features -- -D warnings
-cargo test -- -Z unstable-options --format json --report-time | cargo2junit > results.xml
+RUSTC_BOOTSTRAP=1 cargo test -- -Z unstable-options --format json --report-time | cargo2junit > results.xml
 exit ${PIPESTATUS[0]}

src/anonymiser.rs

@@ -70,6 +70,43 @@ mod tests {
 
     #[test]
     fn successfully_transforms() {
+        let result_file_name = "test_files/results_successfully_transforms.sql";
+        assert!(anonymise(
+            "test_files/dump_file.sql".to_string(),
+            result_file_name.to_string(),
+            "test_files/strategy.json".to_string(),
+            None,
+            TransformerOverrides::none(),
+        )
+        .is_ok());
+
+        let db_url = "postgresql://postgres:postgres@localhost";
+        let postgres = format!("{}/postgres", db_url);
+        let mut conn = Client::connect(&postgres, NoTls).expect("expected connection to succeed");
+
+        conn.simple_query("drop database if exists successfully_transforms_test_db")
+            .unwrap();
+        conn.simple_query("create database successfully_transforms_test_db")
+            .unwrap();
+
+        let result = Command::new("psql")
+            .arg(format!("{}/successfully_transforms_test_db", db_url))
+            .arg("-f")
+            .arg(result_file_name)
+            .arg("-v")
+            .arg("ON_ERROR_STOP=1")
+            .output()
+            .expect("failed!");
+
+        assert!(
+            result.status.success(),
+            "failed to restore backup:\n{:?}",
+            String::from_utf8(result.stderr).unwrap()
+        );
+    }
+
+    #[test]
+    fn successfully_truncates() {
         assert!(anonymise(
             "test_files/dump_file.sql".to_string(),
             "test_files/results.sql".to_string(),
@@ -83,13 +120,15 @@ mod tests {
         let postgres = format!("{}/postgres", db_url);
         let mut conn = Client::connect(&postgres, NoTls).expect("expected connection to succeed");
 
-        conn.simple_query("drop database if exists anonymiser_test")
+        conn.simple_query("drop database if exists successfully_truncates_db_name")
             .unwrap();
-        conn.simple_query("create database anonymiser_test")
+        conn.simple_query("create database successfully_truncates_db_name")
             .unwrap();
 
+        conn.close().expect("expected connection to close");
+
         let result = Command::new("psql")
-            .arg(format!("{}/anonymiser_test", db_url))
+            .arg(format!("{}/successfully_truncates_db_name", db_url))
             .arg("-f")
             .arg("test_files/results.sql")
             .arg("-v")
@@ -102,5 +141,16 @@ mod tests {
             "failed to restore backup:\n{:?}",
             String::from_utf8(result.stderr).unwrap()
         );
+
+        let test_db = format!("{}/successfully_truncates_db_name", db_url);
+        let mut test_db_conn =
+            Client::connect(&test_db, NoTls).expect("expected connection to succeed");
+
+        let extra_data_row_count: i64 = test_db_conn
+            .query_one("select count(*) from extra_data", &[])
+            .unwrap()
+            .get(0);
+
+        assert_eq!(extra_data_row_count, 0);
     }
 }

src/file_reader.rs

@@ -98,6 +98,12 @@ mod tests {
                 strategy_tuple("phone_number"),
             ]),
         );
+
+        strategies.insert(
+            "public.extra_data".to_string(),
+            HashMap::from([strategy_tuple("id"), strategy_tuple("data")]),
+        );
+
         strategies
     }
 
@@ -123,7 +129,8 @@ mod tests {
     fn can_read_and_output_compressed_with_default() {
         let input_file = "test_files/dump_file.sql".to_string();
         let compressed_file = "test_files/compressed_file_reader_test_results.sql".to_string();
-        let uncompressed_file_name = "test_files/uncompressed_file_reader_test_results.sql";
+        let uncompressed_file_name =
+            "test_files/uncompressed_file_reader_can_read_and_output_compressed_with_default.sql";
 
         let _ = fs::remove_file(&compressed_file);
         let _ = fs::remove_file(uncompressed_file_name);
@@ -156,7 +163,7 @@ mod tests {
     fn can_read_and_output_compressed_with_specific_compression_type() {
         let input_file = "test_files/dump_file.sql".to_string();
         let compressed_file = "test_files/compressed_file_reader_test_results.sql".to_string();
-        let uncompressed_file_name = "test_files/uncompressed_file_reader_test_results.sql";
+        let uncompressed_file_name = "test_files/uncompressed_file_reader_can_read_and_output_compressed_with_sepcific_compression_type.sql";
 
         let _ = fs::remove_file(&compressed_file);
         let _ = fs::remove_file(uncompressed_file_name);

src/fixers/db_mismatch.rs

@@ -13,12 +13,15 @@ pub fn fix(
 }
 
 fn add_missing(current: Vec<StrategyInFile>, missing: &[SimpleColumn]) -> Vec<StrategyInFile> {
-    let missing_columns_by_table = missing.iter().fold(HashMap::new(), |mut acc, column| {
-        acc.entry(column.table_name.clone())
-            .or_insert_with(Vec::new)
-            .push(column.column_name.clone());
-        acc
-    });
+    let missing_columns_by_table = missing.iter().fold(
+        HashMap::new(),
+        |mut acc: HashMap<std::string::String, Vec<String>>, column| {
+            acc.entry(column.table_name.clone())
+                .or_default()
+                .push(column.column_name.clone());
+            acc
+        },
+    );
 
     let mut new_strategies = current;
 
@@ -32,6 +35,7 @@ fn add_missing(current: Vec<StrategyInFile>, missing: &[SimpleColumn]) -> Vec<St
             }
             None => {
                 let mut new_table = StrategyInFile {
+                    truncate: false,
                     table_name: table.clone(),
                     description: "".to_string(),
                     columns: vec![],
@@ -51,14 +55,15 @@ fn remove_redundant(
     existing: Vec<StrategyInFile>,
     redundant_columns_to_remove: &[SimpleColumn],
 ) -> Vec<StrategyInFile> {
-    let table_names = redundant_columns_to_remove
-        .iter()
-        .fold(HashMap::new(), |mut acc, column| {
+    let table_names = redundant_columns_to_remove.iter().fold(
+        HashMap::new(),
+        |mut acc: HashMap<std::string::String, Vec<String>>, column| {
             acc.entry(column.table_name.clone())
-                .or_insert_with(Vec::new)
+                .or_default()
                 .push(column.column_name.clone());
             acc
-        });
+        },
+    );
 
     existing
         .into_iter()
@@ -95,6 +100,7 @@ mod tests {
         let current = vec![StrategyInFile {
             table_name: "public.person".to_string(),
             description: "".to_string(),
+            truncate: false,
             columns: vec![ColumnInFile::new("id"), ColumnInFile::new("first_name")],
         }];
 
@@ -119,6 +125,7 @@ mod tests {
             StrategyInFile {
                 table_name: "public.person".to_string(),
                 description: "".to_string(),
+                truncate: false,
                 columns: vec![
                     ColumnInFile::new("id"),
                     ColumnInFile::new("first_name"),
@@ -128,6 +135,7 @@ mod tests {
             StrategyInFile {
                 table_name: "public.location".to_string(),
                 description: "".to_string(),
+                truncate: false,
                 columns: vec![ColumnInFile::new("id"), ColumnInFile::new("post_code")],
             },
         ];
@@ -141,11 +149,13 @@ mod tests {
             StrategyInFile {
                 table_name: "public.location".to_string(),
                 description: "".to_string(),
+                truncate: false,
                 columns: vec![ColumnInFile::new("id"), ColumnInFile::new("post_code")],
             },
             StrategyInFile {
                 table_name: "public.person".to_string(),
                 description: "".to_string(),
+                truncate: false,
                 columns: vec![
                     ColumnInFile::new("id"),
                     ColumnInFile::new("first_name"),
@@ -174,6 +184,7 @@ mod tests {
         let expected = vec![StrategyInFile {
             table_name: "public.person".to_string(),
             description: "".to_string(),
+            truncate: false,
             columns: vec![ColumnInFile::new("id"), ColumnInFile::new("first_name")],
         }];
 

src/main.rs

@@ -12,6 +12,7 @@ use crate::opts::{Anonymiser, Opts};
 use crate::parsers::strategies::Strategies;
 use crate::parsers::strategy_errors::StrategyFileError;
 use crate::parsers::strategy_structs::{StrategyInFile, TransformerOverrides};
+use colored::Colorize;
 use native_tls::TlsConnector;
 use postgres_native_tls::MakeTlsConnector;
 
@@ -57,30 +58,35 @@ fn main() -> Result<(), std::io::Error> {
         Anonymiser::CheckStrategies {
             strategy_file,
             db_url,
-        } => {
-            let strategies = strategy_file::read(&strategy_file).unwrap_or_else(|_| Vec::new());
-
-            match strategy_differences(strategies, db_url) {
+        } => match read_strategy_file(&strategy_file, &db_url) {
+            Ok(strategies) => match strategy_differences(strategies, db_url.clone()) {
                 Ok(()) => println!("All up to date"),
                 Err(err) => {
                     println!("{}", err);
                     if fixer::can_fix(&err) {
-                        println!("But the great news is we can fix at least some of your mess... try running with \"fix-strategies\"");
+                        let retry_command = format!(
+                            "anonymiser fix-strategies --db-url={} --strategy-file={}",
+                            db_url, strategy_file
+                        )
+                        .green();
+                        println!("But the great news is we can fix at least some of your mess... try running:\n{}", retry_command);
                     } else {
                         println!("Bad news... we currently cannot fix this for you, you'll have to sort it out yourself!");
                     }
                     std::process::exit(1);
                 }
+            },
+            Err(err) => {
+                println!("{}", err);
+                std::process::exit(1);
             }
-        }
+        },
 
         Anonymiser::FixStrategies {
             strategy_file,
             db_url,
-        } => {
-            let strategies = strategy_file::read(&strategy_file).unwrap_or_else(|_| Vec::new());
-
-            match strategy_differences(strategies, db_url) {
+        } => match read_strategy_file(&strategy_file, &db_url) {
+            Ok(strategies) => match strategy_differences(strategies, db_url) {
                 Ok(()) => match fixer::just_sort(&strategy_file) {
                     SortResult::Sorted => {
                         println!("Ok, we've updated that for you, check your diff!")
@@ -95,8 +101,12 @@ fn main() -> Result<(), std::io::Error> {
                     fixer::fix(&strategy_file, err);
                     println!("All done, you probably want to run \"check-strategies\" again to make sure");
                 }
+            },
+            Err(err) => {
+                println!("{}", err);
+                std::process::exit(1);
             }
-        }
+        },
 
         Anonymiser::GenerateStrategies {
             strategy_file,
@@ -121,6 +131,23 @@ fn main() -> Result<(), std::io::Error> {
     Ok(())
 }
 
+fn read_strategy_file(strategy_file: &str, db_url: &str) -> Result<Vec<StrategyInFile>, String> {
+    match strategy_file::read(strategy_file) {
+        Ok(strategies) => Ok(strategies),
+        Err(_) => {
+            let retry_command = format!(
+                "anonymiser generate-strategies --db-url={} --strategy-file={}",
+                db_url, strategy_file
+            )
+            .green();
+            Err(format!(
+                "Strategy file {} not found. You can use \n{}\nto create an initial file",
+                strategy_file, retry_command
+            ))
+        }
+    }
+}
+
 fn strategy_differences(
     strategies: Vec<StrategyInFile>,
     db_url: String,