Write out raw bytes, not human-readable u8 arrays.

README.md

@@ -10,9 +10,9 @@ Note that this repository contains _only_ the code needed to generate a commitme
 To run the built-in example through our library, run `cargo run --release --bin example_hyrax_commit`. The `main` function computes the commitment to a mock iris image of size 2^17 (=128 x 1024). It also demonstrates binary serialization/deserialization, and writes/reads the byte stream to/from file.
 
 ### Example binary usage 
-In `./examples`, we've included a shell script `run_hyrax_commit` which will execute our binary using a random iris image found in `./examples/e2etesting/normalized-iris-image.json`. You can generate the commitment
+In `./examples`, we've included a shell script `run_hyrax_commit` which will execute our binary using a dummy image found in `./examples/dummy-data/left_normalized_image.bin`. You can generate the commitment
 and blinding factors for this commitment and write them to file by running `./run_hyrax_commit` within the
-`./examples` directory. The commitment will get written to `./examples/e2etesting/commitment-iris-image-example.json` and the blinding factors will get written to `e2etesting/blinding-factors-iris-image-example.json`.
+`./examples` directory. The commitment will get written to `./examples/dummy-data/left_normalized_image_commitment.bin` and the blinding factors will get written to `dummy-data/left_normalized_image_blinding_factors.bin`.
 
 ## Production Usage
 The primary user-friendly function can be found in `./src/iriscode_commit/mod.rs` as the `compute_commitments_binary_outputs` function. The function takes in as input

examples/dummy-data/.gitignore

@@ -0,0 +1,2 @@
+*commitment*.bin
+*blinding_factors*.bin

examples/run_hyrax_commit

@@ -1,5 +1,9 @@
 cargo build --release
-cargo run --release --bin hyrax_commit -- \
-    --input-image-filepath e2etesting/image-example.json \
-    --output-commitment-filepath e2etesting/commitment-iris-image-example.json \
-    --output-blinding-factors-filepath e2etesting/blinding-factors-iris-image-example.json \
+for suffix in "" "_resized"; do
+    for type in "image" "mask"; do
+        cargo run --release --bin hyrax_commit -- \
+            --input-image-filepath dummy-data/left_normalized_${type}${suffix}.bin \
+            --output-commitment-filepath dummy-data/left_normalized_${type}_commitment${suffix}.bin \
+            --output-blinding-factors-filepath dummy-data/left_normalized_${type}_blinding_factors${suffix}.bin
+    done
+done

src/bin/example_hyrax_commit.rs

@@ -1,47 +1,15 @@
 /// Measure how long it takes to commit to the Worldcoin iris image.
 /// Random u8 values are used as a stand in for the normalized iris image.
 use hyrax::iriscode_commit::{compute_commitments_binary_outputs, HyraxCommitmentOutputSerialized};
-use itertools::Itertools;
+use hyrax::utils::{read_bytes_from_file, write_bytes_to_file, INPUT_NORMALIZED_IMAGE_FILENAME, COMMITMENT_FILENAME, BLINDING_FACTORS_FILENAME};
 use rand::RngCore;
 use rand_core::OsRng;
-use std::fs;
-use std::io::{BufWriter, Read};
 use std::time::Instant;
 
-// image is 128 x 1024 = 2^17 in size
-const LOG_IMAGE_SIZE: usize = 17;
-// this is the file that the image is stored in as an array of bytes. in the example
-// function, we create a random "image" and just save this to file.
-const INPUT_NORMALIZED_IMAGE_FILENAME: &str = "examples/e2etesting/image-example.json";
-// this is the file that the serialized commitment to the iris image is stored in.
-const COMMITMENT_FILENAME: &str = "examples/e2etesting/commit-test1.json";
-// this is the file that the serialized blinding factors are stored in.
-const BLINDING_FACTORS_FILENAME: &str = "examples/e2etesting/bf-test1.json";
-
-/// Helper function for buffered writing to file.
-fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
-    let file = fs::File::create(filename).unwrap();
-    let bw = BufWriter::new(file);
-    serde_json::to_writer(bw, &bytes).unwrap();
-}
-
-/// Helper function for buffered reading from file.
-fn read_bytes_from_file(filename: &str) -> Vec<u8> {
-    let mut file = std::fs::File::open(filename).unwrap();
-    let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
-    let mut bufreader = Vec::with_capacity(initial_buffer_size);
-    file.read_to_end(&mut bufreader).unwrap();
-    serde_json::de::from_slice(&bufreader[..]).unwrap()
-}
-
 /// Usage: `cargo run --release` from this directory (remainder-hyrax-tfh/hyrax/src/bin)
 fn main() {
-    // Generate a random image to be committed to; this is a stand-in for the iris image ---
-    let iris_image = (0..1 << LOG_IMAGE_SIZE)
-        .map(|_| rand::random::<u8>())
-        .collect_vec();
-
-    write_bytes_to_file(INPUT_NORMALIZED_IMAGE_FILENAME, &iris_image);
+    // Read a dummy image from file
+    let iris_image = read_bytes_from_file(INPUT_NORMALIZED_IMAGE_FILENAME);
 
     let start_time = Instant::now();
 

src/bin/hyrax_commit.rs

@@ -6,8 +6,8 @@ use hyrax::utils::{read_bytes_from_file, write_bytes_to_file};
 use rand::RngCore;
 use rand_core::OsRng;
 
-// image is 128 x 1024 = 2^17 in size
-const LOG_IMAGE_SIZE: usize = 17;
+const V2_IMAGE_SIZE: usize = 100 * 400;
+const V3_IMAGE_SIZE: usize = 128 * 1024;
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -33,7 +33,7 @@ fn main() {
     // Generate a random image to be committed to; this is a stand-in for the iris image ---
     let iris_image = read_bytes_from_file(&args.input_image_filepath);
     // Sanity check on expected image dimensions
-    assert_eq!(iris_image.len(), 1 << LOG_IMAGE_SIZE);
+    assert!((iris_image.len() == V2_IMAGE_SIZE) || (iris_image.len() == V3_IMAGE_SIZE));
 
     // Sample randomness for the generation of the blinding factors (note that `OsRng` calls `/dev/urandom` under the hood)
     let mut seed = [0u8; 32];

src/iriscode_commit/tests.rs

@@ -1,9 +1,6 @@
 #[test]
 fn test_serialize_end_to_end() {
-    /// Imports
-    use std::fs;
-    use std::io::{BufWriter, Read};
-
+    use crate::utils::{read_bytes_from_file, write_bytes_to_file, INPUT_NORMALIZED_IMAGE_FILENAME, COMMITMENT_FILENAME, BLINDING_FACTORS_FILENAME};
     use crate::iriscode_commit::{
         compute_commitments, deserialize_blinding_factors_from_bytes_compressed_concrete,
         deserialize_commitment_from_bytes_compressed_concrete, HyraxCommitmentOutput, LOG_NUM_COLS,
@@ -20,31 +17,8 @@ fn test_serialize_end_to_end() {
     use rand::RngCore;
     use rand_core::OsRng;
 
-    /// Helper function for buffered writing to file.
-    fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
-        let file = fs::File::create(filename).unwrap();
-        let bw = BufWriter::new(file);
-        serde_json::to_writer(bw, &bytes).unwrap();
-    }
-
-    /// Helper function for buffered reading from file.
-    fn read_bytes_from_file(filename: &str) -> Vec<u8> {
-        let mut file = std::fs::File::open(filename).unwrap();
-        let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
-        let mut bufreader = Vec::with_capacity(initial_buffer_size);
-        file.read_to_end(&mut bufreader).unwrap();
-        serde_json::de::from_slice(&bufreader[..]).unwrap()
-    }
-
-    // image is 128 x 1024 = 2^17 in size
-    const LOG_IMAGE_SIZE: usize = 17;
-    const TEST_COMMITMENT_FILENAME: &str = "test-commitment-iris-image.json";
-    const TEST_BLINDING_FACTORS_FILENAME: &str = "test-blinding-factors-iris-image.json";
-
-    // --- Generate a random image to be committed to; this is a stand-in for the iris image ---
-    let iris_image = (0..1 << LOG_IMAGE_SIZE)
-        .map(|_| rand::random::<u8>())
-        .collect_vec();
+    // Read a dummy image from file
+    let iris_image = read_bytes_from_file(INPUT_NORMALIZED_IMAGE_FILENAME);
 
     let start_time = Instant::now();
 
@@ -75,12 +49,12 @@ fn test_serialize_end_to_end() {
         .collect_vec();
 
     // --- Sample serialization to file (iris image, blinding factors) ---
-    write_bytes_to_file(TEST_COMMITMENT_FILENAME, &commitment_serialized);
-    write_bytes_to_file(TEST_BLINDING_FACTORS_FILENAME, &blinding_factors_serialized);
+    write_bytes_to_file(COMMITMENT_FILENAME, &commitment_serialized);
+    write_bytes_to_file(BLINDING_FACTORS_FILENAME, &blinding_factors_serialized);
 
     // --- Sample serialization from file (iris image, blinding factors) ---
-    let commitment_bytes_from_file = read_bytes_from_file(TEST_COMMITMENT_FILENAME);
-    let blinding_factors_bytes_from_file = read_bytes_from_file(TEST_BLINDING_FACTORS_FILENAME);
+    let commitment_bytes_from_file = read_bytes_from_file(COMMITMENT_FILENAME);
+    let blinding_factors_bytes_from_file = read_bytes_from_file(BLINDING_FACTORS_FILENAME);
 
     // --- Sanitycheck vs. bytes ---
     assert_eq!(commitment_serialized, commitment_bytes_from_file);

src/utils/mod.rs

@@ -1,26 +1,33 @@
 use std::{
     fs,
-    io::{BufWriter, Read},
+    io::{BufWriter, Read, Write},
 };
 
+/// The file that the image is stored in as an array of bytes.
+pub const INPUT_NORMALIZED_IMAGE_FILENAME: &str = "examples/dummy-data/left_normalized_image.bin";
+/// The file that the serialized commitment to the iris image is stored in.
+pub const COMMITMENT_FILENAME: &str = "examples/dummy-data/left_normalized_image_commitment.bin";
+/// The file that the serialized blinding factors are stored in.
+pub const BLINDING_FACTORS_FILENAME: &str = "examples/dummy-data/left_normalized_image_blinding_factors.bin";
+
 use rand::RngCore;
 use sha3::digest::XofReader;
 use sha3::Sha3XofReader;
 
-/// Helper function for buffered writing to file.
+/// Helper function for buffered writing to file.  Writes raw binary data.
 pub fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
     let file = fs::File::create(filename).unwrap();
-    let bw = BufWriter::new(file);
-    serde_json::to_writer(bw, &bytes).unwrap();
+    let mut bw = BufWriter::new(file);
+    bw.write_all(bytes).unwrap();
 }
 
-/// Helper function for buffered reading from file.
+/// Helper function to read (raw binary) bytes from a file, preallocating the required space.
 pub fn read_bytes_from_file(filename: &str) -> Vec<u8> {
     let mut file = std::fs::File::open(filename).unwrap();
     let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
     let mut bufreader = Vec::with_capacity(initial_buffer_size);
     file.read_to_end(&mut bufreader).unwrap();
-    serde_json::de::from_slice(&bufreader[..]).unwrap()
+    bufreader
 }
 
 pub struct Sha3XofReaderWrapper {