Write out raw bytes, not human-readable u8 arrays. (#6)

* Read and write raw JSON. Include dummy images of v2 & v3 dimensions

* Ran cargo fmt

* Fixed all clippy warnings

---------

Co-authored-by: Ryan Cao <[email protected]>

README.md

@@ -10,9 +10,9 @@ Note that this repository contains _only_ the code needed to generate a commitme
 To run the built-in example through our library, run `cargo run --release --bin example_hyrax_commit`. The `main` function computes the commitment to a mock iris image of size 2^17 (=128 x 1024). It also demonstrates binary serialization/deserialization, and writes/reads the byte stream to/from file.
 
 ### Example binary usage 
-In `./examples`, we've included a shell script `run_hyrax_commit` which will execute our binary using a random iris image found in `./examples/e2etesting/normalized-iris-image.json`. You can generate the commitment
+In `./examples`, we've included a shell script `run_hyrax_commit` which will execute our binary using a dummy image found in `./examples/dummy-data/left_normalized_image.bin`. You can generate the commitment
 and blinding factors for this commitment and write them to file by running `./run_hyrax_commit` within the
-`./examples` directory. The commitment will get written to `./examples/e2etesting/commitment-iris-image-example.json` and the blinding factors will get written to `e2etesting/blinding-factors-iris-image-example.json`.
+`./examples` directory. The commitment will get written to `./examples/dummy-data/left_normalized_image_commitment.bin` and the blinding factors will get written to `dummy-data/left_normalized_image_blinding_factors.bin`.
 
 ## Production Usage
 The primary user-friendly function can be found in `./src/iriscode_commit/mod.rs` as the `compute_commitments_binary_outputs` function. The function takes in as input

examples/dummy-data/.gitignore

@@ -0,0 +1,2 @@
+*commitment*.bin
+*blinding_factors*.bin

examples/run_hyrax_commit

@@ -1,5 +1,9 @@
 cargo build --release
-cargo run --release --bin hyrax_commit -- \
-    --input-image-filepath e2etesting/image-example.json \
-    --output-commitment-filepath e2etesting/commitment-iris-image-example.json \
-    --output-blinding-factors-filepath e2etesting/blinding-factors-iris-image-example.json \
+for suffix in "" "_resized"; do
+    for type in "image" "mask"; do
+        cargo run --release --bin hyrax_commit -- \
+            --input-image-filepath dummy-data/left_normalized_${type}${suffix}.bin \
+            --output-commitment-filepath dummy-data/left_normalized_${type}_commitment${suffix}.bin \
+            --output-blinding-factors-filepath dummy-data/left_normalized_${type}_blinding_factors${suffix}.bin
+    done
+done

src/bin/example_hyrax_commit.rs

@@ -1,47 +1,18 @@
 /// Measure how long it takes to commit to the Worldcoin iris image.
 /// Random u8 values are used as a stand in for the normalized iris image.
 use hyrax::iriscode_commit::{compute_commitments_binary_outputs, HyraxCommitmentOutputSerialized};
-use itertools::Itertools;
+use hyrax::utils::{
+    read_bytes_from_file, write_bytes_to_file, BLINDING_FACTORS_FILENAME, COMMITMENT_FILENAME,
+    INPUT_NORMALIZED_IMAGE_FILENAME,
+};
 use rand::RngCore;
 use rand_core::OsRng;
-use std::fs;
-use std::io::{BufWriter, Read};
 use std::time::Instant;
 
-// image is 128 x 1024 = 2^17 in size
-const LOG_IMAGE_SIZE: usize = 17;
-// this is the file that the image is stored in as an array of bytes. in the example
-// function, we create a random "image" and just save this to file.
-const INPUT_NORMALIZED_IMAGE_FILENAME: &str = "examples/e2etesting/image-example.json";
-// this is the file that the serialized commitment to the iris image is stored in.
-const COMMITMENT_FILENAME: &str = "examples/e2etesting/commit-test1.json";
-// this is the file that the serialized blinding factors are stored in.
-const BLINDING_FACTORS_FILENAME: &str = "examples/e2etesting/bf-test1.json";
-
-/// Helper function for buffered writing to file.
-fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
-    let file = fs::File::create(filename).unwrap();
-    let bw = BufWriter::new(file);
-    serde_json::to_writer(bw, &bytes).unwrap();
-}
-
-/// Helper function for buffered reading from file.
-fn read_bytes_from_file(filename: &str) -> Vec<u8> {
-    let mut file = std::fs::File::open(filename).unwrap();
-    let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
-    let mut bufreader = Vec::with_capacity(initial_buffer_size);
-    file.read_to_end(&mut bufreader).unwrap();
-    serde_json::de::from_slice(&bufreader[..]).unwrap()
-}
-
 /// Usage: `cargo run --release` from this directory (remainder-hyrax-tfh/hyrax/src/bin)
 fn main() {
-    // Generate a random image to be committed to; this is a stand-in for the iris image ---
-    let iris_image = (0..1 << LOG_IMAGE_SIZE)
-        .map(|_| rand::random::<u8>())
-        .collect_vec();
-
-    write_bytes_to_file(INPUT_NORMALIZED_IMAGE_FILENAME, &iris_image);
+    // Read a dummy image from file
+    let iris_image = read_bytes_from_file(INPUT_NORMALIZED_IMAGE_FILENAME);
 
     let start_time = Instant::now();
 

src/bin/hyrax_commit.rs

@@ -6,8 +6,8 @@ use hyrax::utils::{read_bytes_from_file, write_bytes_to_file};
 use rand::RngCore;
 use rand_core::OsRng;
 
-// image is 128 x 1024 = 2^17 in size
-const LOG_IMAGE_SIZE: usize = 17;
+const V2_IMAGE_SIZE: usize = 100 * 400;
+const V3_IMAGE_SIZE: usize = 128 * 1024;
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -33,7 +33,7 @@ fn main() {
     // Generate a random image to be committed to; this is a stand-in for the iris image ---
     let iris_image = read_bytes_from_file(&args.input_image_filepath);
     // Sanity check on expected image dimensions
-    assert_eq!(iris_image.len(), 1 << LOG_IMAGE_SIZE);
+    assert!((iris_image.len() == V2_IMAGE_SIZE) || (iris_image.len() == V3_IMAGE_SIZE));
 
     // Sample randomness for the generation of the blinding factors (note that `OsRng` calls `/dev/urandom` under the hood)
     let mut seed = [0u8; 32];

src/curves/mod.rs

@@ -116,11 +116,7 @@ impl PrimeOrderCurve for Bn256Point {
             true
         } else {
             let (x, y) = self.affine_coordinates().unwrap();
-            if ((x * x + Bn256Point::a()) * x + Bn256Point::b()) == y {
-                true
-            } else {
-                false
-            }
+            ((x * x + Bn256Point::a()) * x + Bn256Point::b()) == y
         }
     }
 
@@ -157,7 +153,7 @@ impl PrimeOrderCurve for Bn256Point {
     }
 
     fn double(&self) -> Self {
-        Group::double(&self)
+        Group::double(self)
     }
 
     fn projective_coordinates(&self) -> (Self::Base, Self::Base, Self::Base) {
@@ -189,7 +185,7 @@ impl PrimeOrderCurve for Bn256Point {
     /// The bytestring representation of the BN256 curve is a `[u8; 65]` with
     /// the following semantic representation:
     /// * The first `u8` byte represents whether the point is a point at
-    /// infinity (in affine coordinates). 1 if it is at infinity, 0 otherwise.
+    ///     infinity (in affine coordinates). 1 if it is at infinity, 0 otherwise.
     /// * The next 32 `u8` bytes represent the x-coordinate of the point in little endian.
     /// * The next 32 `u8` bytes represent the y-coordinate of the point in little endian.
     fn to_bytes_uncompressed(&self) -> Vec<u8> {
@@ -200,24 +196,24 @@ impl PrimeOrderCurve for Bn256Point {
             let x_bytes = x.into_bigint().to_bytes_le();
             let y_bytes = y.into_bigint().to_bytes_le();
             let all_bytes = std::iter::once(0_u8)
-                .chain(x_bytes.into_iter())
-                .chain(y_bytes.into_iter())
+                .chain(x_bytes)
+                .chain(y_bytes)
                 .collect_vec();
             assert_eq!(all_bytes.len(), Self::UNCOMPRESSED_CURVE_POINT_BYTEWIDTH);
             all_bytes
         } else {
             // --- Point at infinity ---
-            return [1_u8; 65].to_vec();
+            [1_u8; 65].to_vec()
         }
     }
 
     /// The bytestring representation of the BN256 curve is a `[u8; 34]` with
     /// the following semantic representation:
     /// * The first `u8` byte represents whether the point is a point at
-    /// infinity (in affine coordinates).
+    ///     infinity (in affine coordinates).
     /// * The next 32 `u8` bytes represent the x-coordinate of the point in little endian.
     /// * The final `u8` byte represents the sign of the y-coordinate of the
-    /// point.
+    ///     point.
     fn to_bytes_compressed(&self) -> Vec<u8> {
         // --- First get the affine coordinates. If `None`, we have a point at infinity. ---
         let affine_coords = self.affine_coordinates();
@@ -230,14 +226,14 @@ impl PrimeOrderCurve for Bn256Point {
             // the field modulus is odd.
             let y_parity = y.into_bigint().to_bytes_le()[0] & 1;
             let all_bytes = std::iter::once(0_u8)
-                .chain(x_bytes.into_iter())
+                .chain(x_bytes)
                 .chain(std::iter::once(y_parity))
                 .collect_vec();
             assert_eq!(all_bytes.len(), Self::COMPRESSED_CURVE_POINT_BYTEWIDTH);
             all_bytes
         } else {
             // --- Point at infinity ---
-            return [1_u8; 34].to_vec();
+            [1_u8; 34].to_vec()
         }
     }
 
@@ -249,11 +245,11 @@ impl PrimeOrderCurve for Bn256Point {
         assert_eq!(bytes.len(), Self::UNCOMPRESSED_CURVE_POINT_BYTEWIDTH);
         // first check if it is a point at infinity
         if bytes[0] == 1_u8 {
-            return Self {
+            Self {
                 x: Self::Base::zero(),
                 y: Self::Base::one(),
                 z: Self::Base::zero(),
-            };
+            }
         } else {
             let mut x_bytes_alloc = [0_u8; 32];
             let x_bytes = &bytes[1..33];
@@ -285,11 +281,11 @@ impl PrimeOrderCurve for Bn256Point {
         assert_eq!(bytes.len(), Self::COMPRESSED_CURVE_POINT_BYTEWIDTH);
         // first check if it is a point at infinity
         if bytes[0] == 1_u8 {
-            return Self {
+            Self {
                 x: Self::Base::zero(),
                 y: Self::Base::one(),
                 z: Self::Base::zero(),
-            };
+            }
         } else {
             let y_sign_byte: u8 = bytes[33];
 
@@ -304,13 +300,11 @@ impl PrimeOrderCurve for Bn256Point {
                 y_option_2
             };
 
-            let point = Self {
+            Self {
                 x: x_coord,
                 y: y_coord,
                 z: Self::Base::one(),
-            };
-
-            point
+            }
         }
     }
 }

src/iriscode_commit/mod.rs

@@ -18,10 +18,10 @@ pub const PUBLIC_STRING: &str = "Modulus <3 Worldcoin: ZKML Self-Custody Edition
 
 /// The Hyrax polynomial commitment scheme returns two things:
 /// * The `commitment` itself, to be signed by the Orb and sent to Worldcoin's
-/// backend (i.e. the verifier), and
+///     backend (i.e. the verifier), and
 /// * The `blinding_factors`, to be sent in the clear to ONLY the user's device
-/// (leaking these to anyone else will cause the commitment to leak information
-/// about the user's iris scan)
+///     (leaking these to anyone else will cause the commitment to leak information
+///     about the user's iris scan)
 pub struct HyraxCommitmentOutput<C: PrimeOrderCurve> {
     pub commitment: Vec<C>,
     pub blinding_factors: Vec<C::Scalar>,
@@ -106,7 +106,7 @@ pub fn compute_commitments<C: PrimeOrderCurve>(
     let row_chunks = data_vec.chunks(n_cols);
     let commitment = row_chunks
         .zip(blinding_factors.iter())
-        .map(|(chunk, blind)| vector_committer.vector_commit(&chunk, blind))
+        .map(|(chunk, blind)| vector_committer.vector_commit(chunk, blind))
         .collect_vec();
 
     HyraxCommitmentOutput {
@@ -129,7 +129,7 @@ pub fn deserialize_blinding_factors_from_bytes_compressed<C: PrimeOrderCurve>(
 ) -> Vec<C::Scalar> {
     let blinding_factors: Vec<<C as PrimeOrderCurve>::Scalar> = bytes
         .chunks(C::SCALAR_ELEM_BYTEWIDTH)
-        .map(|byte_repr| C::Scalar::from_le_bytes_mod_order(byte_repr))
+        .map(C::Scalar::from_le_bytes_mod_order)
         .collect_vec();
     blinding_factors
 }

src/iriscode_commit/tests.rs

@@ -1,15 +1,15 @@
 #[test]
 fn test_serialize_end_to_end() {
-    /// Imports
-    use std::fs;
-    use std::io::{BufWriter, Read};
-
     use crate::iriscode_commit::{
         compute_commitments, deserialize_blinding_factors_from_bytes_compressed_concrete,
         deserialize_commitment_from_bytes_compressed_concrete, HyraxCommitmentOutput, LOG_NUM_COLS,
         PUBLIC_STRING,
     };
     use crate::pedersen::PedersenCommitter;
+    use crate::utils::{
+        read_bytes_from_file, write_bytes_to_file, BLINDING_FACTORS_FILENAME, COMMITMENT_FILENAME,
+        INPUT_NORMALIZED_IMAGE_FILENAME,
+    };
     use std::time::Instant;
 
     use crate::curves::PrimeOrderCurve;
@@ -20,31 +20,8 @@ fn test_serialize_end_to_end() {
     use rand::RngCore;
     use rand_core::OsRng;
 
-    /// Helper function for buffered writing to file.
-    fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
-        let file = fs::File::create(filename).unwrap();
-        let bw = BufWriter::new(file);
-        serde_json::to_writer(bw, &bytes).unwrap();
-    }
-
-    /// Helper function for buffered reading from file.
-    fn read_bytes_from_file(filename: &str) -> Vec<u8> {
-        let mut file = std::fs::File::open(filename).unwrap();
-        let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
-        let mut bufreader = Vec::with_capacity(initial_buffer_size);
-        file.read_to_end(&mut bufreader).unwrap();
-        serde_json::de::from_slice(&bufreader[..]).unwrap()
-    }
-
-    // image is 128 x 1024 = 2^17 in size
-    const LOG_IMAGE_SIZE: usize = 17;
-    const TEST_COMMITMENT_FILENAME: &str = "test-commitment-iris-image.json";
-    const TEST_BLINDING_FACTORS_FILENAME: &str = "test-blinding-factors-iris-image.json";
-
-    // --- Generate a random image to be committed to; this is a stand-in for the iris image ---
-    let iris_image = (0..1 << LOG_IMAGE_SIZE)
-        .map(|_| rand::random::<u8>())
-        .collect_vec();
+    // Read a dummy image from file
+    let iris_image = read_bytes_from_file(INPUT_NORMALIZED_IMAGE_FILENAME);
 
     let start_time = Instant::now();
 
@@ -75,12 +52,12 @@ fn test_serialize_end_to_end() {
         .collect_vec();
 
     // --- Sample serialization to file (iris image, blinding factors) ---
-    write_bytes_to_file(TEST_COMMITMENT_FILENAME, &commitment_serialized);
-    write_bytes_to_file(TEST_BLINDING_FACTORS_FILENAME, &blinding_factors_serialized);
+    write_bytes_to_file(COMMITMENT_FILENAME, &commitment_serialized);
+    write_bytes_to_file(BLINDING_FACTORS_FILENAME, &blinding_factors_serialized);
 
     // --- Sample serialization from file (iris image, blinding factors) ---
-    let commitment_bytes_from_file = read_bytes_from_file(TEST_COMMITMENT_FILENAME);
-    let blinding_factors_bytes_from_file = read_bytes_from_file(TEST_BLINDING_FACTORS_FILENAME);
+    let commitment_bytes_from_file = read_bytes_from_file(COMMITMENT_FILENAME);
+    let blinding_factors_bytes_from_file = read_bytes_from_file(BLINDING_FACTORS_FILENAME);
 
     // --- Sanitycheck vs. bytes ---
     assert_eq!(commitment_serialized, commitment_bytes_from_file);

src/pedersen/mod.rs

@@ -75,7 +75,7 @@ impl<C: PrimeOrderCurve> PedersenCommitter<C> {
                 let mut acc = C::zero();
                 bits.into_iter().enumerate().for_each(|(i, bit)| {
                     if bit {
-                        acc = acc + generator_doublings[i];
+                        acc += generator_doublings[i];
                     }
                 });
                 acc

src/utils/mod.rs

@@ -1,26 +1,34 @@
 use std::{
     fs,
-    io::{BufWriter, Read},
+    io::{BufWriter, Read, Write},
 };
 
+/// The file that the image is stored in as an array of bytes.
+pub const INPUT_NORMALIZED_IMAGE_FILENAME: &str = "examples/dummy-data/left_normalized_image.bin";
+/// The file that the serialized commitment to the iris image is stored in.
+pub const COMMITMENT_FILENAME: &str = "examples/dummy-data/left_normalized_image_commitment.bin";
+/// The file that the serialized blinding factors are stored in.
+pub const BLINDING_FACTORS_FILENAME: &str =
+    "examples/dummy-data/left_normalized_image_blinding_factors.bin";
+
 use rand::RngCore;
 use sha3::digest::XofReader;
 use sha3::Sha3XofReader;
 
-/// Helper function for buffered writing to file.
+/// Helper function for buffered writing to file.  Writes raw binary data.
 pub fn write_bytes_to_file(filename: &str, bytes: &[u8]) {
     let file = fs::File::create(filename).unwrap();
-    let bw = BufWriter::new(file);
-    serde_json::to_writer(bw, &bytes).unwrap();
+    let mut bw = BufWriter::new(file);
+    bw.write_all(bytes).unwrap();
 }
 
-/// Helper function for buffered reading from file.
+/// Helper function to read (raw binary) bytes from a file, preallocating the required space.
 pub fn read_bytes_from_file(filename: &str) -> Vec<u8> {
     let mut file = std::fs::File::open(filename).unwrap();
     let initial_buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
     let mut bufreader = Vec::with_capacity(initial_buffer_size);
     file.read_to_end(&mut bufreader).unwrap();
-    serde_json::de::from_slice(&bufreader[..]).unwrap()
+    bufreader
 }
 
 pub struct Sha3XofReaderWrapper {