//! Sparse proofs generated by Commonware's public storage adapter. use super::*; use commonware_storage::merkle::{ Error, Position, hasher::Hasher as _, storage::Storage, verification, }; use futures::executor::block_on; use std::collections::BTreeMap; type RootProof = (D, Proof); sol! { struct MultiOutput { bytes32 root; bytes32[] elements; bytes32[] proof; uint256 leaves; uint256[] positions; } struct MultiInput { bytes32 root; uint256 leaves; uint256[] locations; bytes32[] elements; bytes32[] proof; uint256[] positions; } } #[derive(Args)] pub(crate) struct MultiArgs { #[arg(long)] leaf_count: u64, /// Comma-separated strictly ascending leaf locations. #[arg(long, value_delimiter = ',', num_args = 1, required = true)] locations: Vec, #[arg(long)] seed: u64, #[command(flatten)] policy: Policy, } impl MultiArgs { pub(super) fn execute( self, family: TreeKind, mode: GenerationMode, ) -> Result, String> { let output = match family { TreeKind::Mmr => self.generate::(mode)?, TreeKind::Mmb => self.generate::(mode)?, }; let (output, positions) = output; Ok(MultiOutput { root: output.root.into(), elements: output.elements.into_iter().map(Into::into).collect(), proof: output.proof.into_iter().map(Into::into).collect(), leaves: Uint256::from(output.leaves), positions: positions.into_iter().map(Uint256::from).collect(), } .abi_encode_params()) } fn generate( &self, mode: GenerationMode, ) -> Result<(Output, Vec), String> { if self.leaf_count > *F::MAX_LEAVES || self.locations.is_empty() || self.locations.len() > 4096 || self.locations.iter().any(|&loc| loc >= self.leaf_count) { return Err("invalid sparse request (maximum 4096 elements)".into()); } let hasher = self.policy.hasher::(); let locations: Vec<_> = self.locations.iter().copied().map(Location::new).collect(); // Planning the witness positions rejects locations that are not strictly ascending // before any tree is built. let positions = canonical_positions::(self.leaf_count, &locations, self.policy)?; let (root, proof) = match mode { GenerationMode::Synthetic => { let tree = SparseTree::new(self.leaf_count, self.seed, &locations, &hasher); prove::(&tree, self.policy, &locations)? } GenerationMode::Materialized { .. } => { let tree = materialize::(self.leaf_count, self.seed, &hasher)?; prove::(&tree, self.policy, &locations)? } }; let elements: Vec<_> = self .locations .iter() .map(|&loc| leaf(self.seed, loc)) .collect(); let pairs: Vec<_> = elements .iter() .zip(&locations) .map(|(e, &loc)| (*e, loc)) .collect(); if !proof.verify_multi_inclusion(&hasher, &pairs, &root) { return Err("Commonware rejected its generated multiproof".into()); } if matches!( mode, GenerationMode::Synthetic | GenerationMode::Materialized { check_mutated: true } ) { check_rejections::(&hasher, &proof, &pairs, root)?; } Ok(( Output { root: root.as_ref().try_into().unwrap(), elements, proof: proof .digests .iter() .map(|d| d.as_ref().try_into().unwrap()) .collect(), leaves: self.leaf_count, }, positions, )) } } fn prove>( tree: &S, policy: Policy, locations: &[Location], ) -> Result, String> { block_on(async { let hasher = policy.hasher::(); let leaves = Location::::try_from(tree.size()).map_err(|error| format!("{error}"))?; let mut peaks = Vec::new(); for (pos, _) in F::peaks(tree.size()) { let digest = tree .get_node(pos) .await .map_err(|error| error.to_string())? .ok_or_else(|| "missing peak digest".to_owned())?; peaks.push(digest); } let root = hasher .root(leaves, policy.inactive_peaks, &peaks) .map_err(|error| error.to_string())?; let proof = verification::multi_proof(tree, policy.inactive_peaks, locations) .await .map_err(|error| error.to_string())?; Ok((root, proof)) }) } fn check_rejections( hasher: &MerkleHasher, proof: &Proof, pairs: &[([u8; 32], Location)], root: H::Digest, ) -> Result<(), String> { let mut bytes = root.as_ref().to_vec(); bytes[0] ^= 1; let bad_root = H::Digest::decode(Copying(bytes.as_slice())).unwrap(); let mut bad_pairs = pairs.to_vec(); bad_pairs[0].0[0] ^= 1; if proof.verify_multi_inclusion(hasher, pairs, &bad_root) || proof.verify_multi_inclusion(hasher, &bad_pairs, &root) { return Err("multiproof accepted a mutated root or element".into()); } if !proof.digests.is_empty() { let mut bad_proof = proof.clone(); let mut bytes = bad_proof.digests[0].as_ref().to_vec(); bytes[0] ^= 1; bad_proof.digests[0] = H::Digest::decode(Copying(bytes.as_slice())).unwrap(); if bad_proof.verify_multi_inclusion(hasher, pairs, &root) { return Err("multiproof accepted a mutated digest".into()); } } Ok(()) } /// A coherent partial tree with arbitrary digests outside the selected paths. /// Family owns positions and child relationships while Commonware owns all hashing. struct SparseTree { size: Position, nodes: BTreeMap, D>, } impl SparseTree { /// Builds the paths to strictly ascending `locations`. fn new>( leaves: u64, seed: u64, locations: &[Location], hasher: &MerkleHasher, ) -> Self { let size = F::location_to_position(Location::new(leaves)); let mut tree = Self { size, nodes: BTreeMap::new(), }; let mut start = 0; for (pos, height) in F::peaks(size) { let end = start + (1u64 << height); let from = locations.partition_point(|loc| **loc < start); let to = locations.partition_point(|loc| **loc < end); tree.subtree(pos, height, seed, &locations[from..to], hasher); start = end; } tree } fn subtree>( &mut self, pos: Position, height: u32, seed: u64, selected: &[Location], hasher: &MerkleHasher, ) -> D { let digest = if selected.is_empty() { D::decode(Copying(leaf(seed ^ u64::MAX, *pos).as_slice())).unwrap() } else if height == 0 { hasher.leaf_digest(pos, &leaf(seed, *selected[0])) } else { let (left, right) = F::children(pos, height); // The first leaf of the right child divides the selected locations. let mut right_leaf = right; for level in (1..height).rev() { right_leaf = F::children(right_leaf, level).0; } let boundary = F::position_to_location(right_leaf).expect("leaf position"); let split = selected.partition_point(|loc| *loc < boundary); let left_digest = self.subtree(left, height - 1, seed, &selected[..split], hasher); let right_digest = self.subtree(right, height - 1, seed, &selected[split..], hasher); hasher.node_digest(pos, &left_digest, &right_digest) }; self.nodes.insert(pos, digest); digest } } impl Storage for SparseTree { type Digest = D; fn size(&self) -> Position { self.size } async fn get_node(&self, position: Position) -> Result, Error> { Ok(self.nodes.get(&position).copied()) } } /// Encodes node positions as digests so Commonware supplies the canonical witness order. struct PositionStore(Position); impl Storage for PositionStore { type Digest = Digest; fn size(&self) -> Position { self.0 } async fn get_node(&self, position: Position) -> Result, Error> { Ok(Some(keccak256::Digest( Uint256::from(*position).to_be_bytes(), ))) } } pub(super) fn canonical_positions( leaves: u64, locations: &[Location], policy: Policy, ) -> Result, String> { if locations.is_empty() { return if leaves == 0 && policy.inactive_peaks == 0 { Ok(vec![]) } else { Err("empty sparse request".into()) }; } let store = PositionStore(F::location_to_position(Location::::new(leaves))); let proof = block_on(verification::multi_proof( &store, policy.inactive_peaks, locations, )) .map_err(|error| error.to_string())?; Ok(proof .digests .iter() .map(|digest| Uint256::from_be_bytes(digest.0).to::()) .collect()) } /// Verifies a canonical ABI `(root, leaves, locations, elements, proof, positions)` tuple. /// Returns false unless the locations are strictly ascending, even when every element is genuine. pub(super) fn check(encoded: &[u8], policy: Policy) -> bool { let Ok(payload) = ::abi_decode_params_with_config( encoded, AbiDecoderConfig::new().strict(true), ) else { return false; }; let Ok(leaves) = u64::try_from(payload.leaves) else { return false; }; if leaves > *F::MAX_LEAVES || payload.locations.len() != payload.elements.len() { return false; } let mut pairs = Vec::with_capacity(payload.elements.len()); for (location, element) in payload.locations.into_iter().zip(payload.elements) { let Ok(loc) = u64::try_from(location) else { return false; }; if loc >= leaves { return false; } pairs.push((element, Location::new(loc))); } let locations: Vec<_> = pairs.iter().map(|(_, loc)| *loc).collect(); let Ok(expected) = canonical_positions::(leaves, &locations, policy) else { return false; }; if payload.positions.len() != expected.len() || payload .positions .iter() .zip(expected) .any(|(position, expected)| *position != Uint256::from(expected)) { return false; } let proof = Proof:: { leaves: Location::new(leaves), inactive_peaks: policy.inactive_peaks, digests: payload .proof .into_iter() .map(|d| H::Digest::decode(Copying(d.as_slice())).unwrap()) .collect(), }; proof.verify_multi_inclusion( &policy.hasher::(), &pairs, &H::Digest::decode(Copying(payload.root.as_slice())).unwrap(), ) } #[cfg(test)] mod tests { use super::*; use crate::Cli; use clap::Parser; fn check_input(output: &Output, locations: &[u64], positions: &[u64]) -> Vec { MultiInput { root: output.root.into(), leaves: Uint256::from(output.leaves), locations: locations.iter().copied().map(Uint256::from).collect(), elements: output.elements.iter().copied().map(Into::into).collect(), proof: output.proof.iter().map(|digest| (*digest).into()).collect(), positions: positions.iter().copied().map(Uint256::from).collect(), } .abi_encode_params() } fn policies() { for bagging in [Fold::Forward, Fold::Backward] { for leaves in [1, 2, 3, 7, 11, 31, 62] { let peak_count = F::peaks(F::location_to_position(Location::new(leaves))).count(); for inactive_peaks in 0..=peak_count { let policy = Policy { bagging, inactive_peaks, }; let locations = if leaves < 3 { (0..leaves).collect() } else { vec![0, leaves / 2, leaves - 1] }; for locations in [vec![0], locations] { let args = MultiArgs { leaf_count: leaves, locations, seed: 42, policy, }; for mode in [ GenerationMode::Materialized { check_mutated: true, }, GenerationMode::Synthetic, ] { let (output, positions) = args.generate::(mode).unwrap(); let encoded = check_input(&output, &args.locations, &positions); assert!(check::(&encoded, policy)); // Locations must be strictly ascending. Checking rejects a repeated // location whether or not its element matches, and rejects a descent. // Generation reports the offending location. let last = args.locations.len() - 1; let mut repeated = ::abi_decode_params_validate(&encoded) .unwrap(); repeated.locations.push(repeated.locations[last]); repeated.elements.push(repeated.elements[last]); assert!(!check::(&repeated.abi_encode_params(), policy)); repeated.elements[last + 1][0] ^= 1; assert!(!check::(&repeated.abi_encode_params(), policy)); let mut locations = args.locations.clone(); locations.push(locations[last]); let duplicate = MultiArgs { leaf_count: leaves, locations, seed: 42, policy, }; assert_eq!( duplicate.generate::(mode).err(), Some( Error::::DuplicateLocation(Location::new( args.locations[last] )) .to_string() ) ); if last != 0 { let mut descending = ::abi_decode_params_validate(&encoded) .unwrap(); descending.locations.reverse(); descending.elements.reverse(); assert!(!check::(&descending.abi_encode_params(), policy)); let mut locations = args.locations.clone(); locations.reverse(); let descending = MultiArgs { leaf_count: leaves, locations, seed: 42, policy, }; assert_eq!( descending.generate::(mode).err(), Some( Error::::UnsortedLocation(Location::new( args.locations[last - 1] )) .to_string() ) ); } let mut bad = encoded.clone(); bad[0] ^= 1; assert!(!check::(&bad, policy)); let mut bad = encoded.clone(); bad[224] = 1; assert!(!check::(&bad, policy)); let mut bad = encoded.clone(); let element_offset = abi_u64(&bad, 96).unwrap() as usize; bad[element_offset..element_offset + 32] .copy_from_slice(&0u64.abi_encode()); assert!(!check::(&bad, policy)); let mut bad = encoded.clone(); let proof_offset = abi_u64(&bad, 128).unwrap() as usize; bad[proof_offset..proof_offset + 32] .copy_from_slice(&(output.proof.len() as u64 + 1).abi_encode()); bad.extend_from_slice(&[0; 32]); assert!(!check::(&bad, policy)); assert!(!check::(&encoded[..encoded.len() - 1], policy)); if !positions.is_empty() { let offset = abi_u64(&encoded, 160).unwrap() as usize; let mut changed = encoded.clone(); changed[offset + 63] ^= 1; assert!(!check::(&changed, policy)); changed = encoded.clone(); changed[offset..offset + 32] .copy_from_slice(&(positions.len() as u64 - 1).abi_encode()); assert!(!check::(&changed, policy)); } let mut wrong_policy = policy; wrong_policy.inactive_peaks = peak_count + 1; assert!(!check::(&encoded, wrong_policy)); } } } } } } #[test] fn sparse_proofs_cover_policies_order_duplicates_and_malformed_abi() { sparse_proofs_cover_policies_order_duplicates_and_malformed_abi_with::(); sparse_proofs_cover_policies_order_duplicates_and_malformed_abi_with::(); } fn sparse_proofs_cover_policies_order_duplicates_and_malformed_abi_with() { policies::(); policies::(); } fn deep() { for leaves in [*F::MAX_LEAVES - 1, *F::MAX_LEAVES] { let peak_count = F::peaks(F::location_to_position(Location::new(leaves))).count(); for bagging in [Fold::Forward, Fold::Backward] { for inactive_peaks in [0, 1, peak_count] { let args = MultiArgs { leaf_count: leaves, locations: vec![0, 1, leaves / 2 - 1, leaves / 2, leaves - 1], seed: 17, policy: Policy { bagging, inactive_peaks, }, }; let (output, positions) = args.generate::(GenerationMode::Synthetic).unwrap(); assert!(check::( &check_input(&output, &args.locations, &positions), args.policy )); } } } } #[test] fn unordered_locations_are_rejected_before_materialization() { // The leaf count exceeds the materialization limit, so these errors come from the // ordering check that runs before the tree is built. fn check() { for (locations, expected) in [ (vec![7, 7], Error::::DuplicateLocation(Location::new(7))), (vec![0, 7, 7], Error::DuplicateLocation(Location::new(7))), (vec![7, 0, 7], Error::UnsortedLocation(Location::new(0))), ] { let args = MultiArgs { leaf_count: 1_000_001, locations, seed: 17, policy: Policy { bagging: Fold::Backward, inactive_peaks: 0, }, }; let error = args .generate::(GenerationMode::Materialized { check_mutated: false, }) .err() .unwrap(); assert_eq!(error, expected.to_string()); } } check::(); check::(); } #[test] fn sparse_proofs_reach_maximum_depth() { sparse_proofs_reach_maximum_depth_with::(); sparse_proofs_reach_maximum_depth_with::(); } fn sparse_proofs_reach_maximum_depth_with() { deep::(); deep::(); } fn complete_selection() { for leaves in [1, 3, 11, 31] { let args = MultiArgs { leaf_count: leaves, locations: (0..leaves).collect(), seed: 81, policy: Policy { bagging: Fold::Backward, inactive_peaks: 1, }, }; let (full, full_positions) = args .generate::(GenerationMode::Materialized { check_mutated: true, }) .unwrap(); let (sparse, sparse_positions) = args.generate::(GenerationMode::Synthetic).unwrap(); assert_eq!(full.root, sparse.root); assert_eq!(full.proof, sparse.proof); assert_eq!(full_positions, sparse_positions); } } #[test] fn complete_sparse_selection_matches_materialized_tree() { complete_sparse_selection_matches_materialized_tree_with::(); complete_sparse_selection_matches_materialized_tree_with::(); } fn complete_sparse_selection_matches_materialized_tree_with() { complete_selection::(); complete_selection::(); } #[test] fn cli_sparse_forms_and_empty_check() { for mode in ["generate-multi", "synthetic-multi"] { for family in ["mmr", "mmb"] { let mut args = vec![ "commonware-sol-fuzz", "merkle", "--hash", "keccak256", mode, "--family", family, "--leaf-count", "31", "--locations", "0,8,12,30", "--seed", "42", "--bagging", "backward", "--inactive-peaks", "2", ]; if mode == "generate-multi" { args.push("--check-mutated"); } let encoded = Cli::try_parse_from(args) .unwrap() .command .execute() .unwrap(); assert!(encoded.len() >= 192); } } for bagging in [Fold::Forward, Fold::Backward] { empty_check::(bagging); empty_check::(bagging); } } /// Checks that an empty multiproof verifies the empty root of family `F` only under `F` and /// `bagging`, and that malformed encodings or a nonzero inactive boundary fail. fn empty_check(bagging: Fold) { let policy = Policy { bagging, inactive_peaks: 0, }; let other = Policy { bagging: match bagging { Fold::Forward => Fold::Backward, Fold::Backward => Fold::Forward, }, inactive_peaks: 0, }; let tags = [ F::DOMAIN, policy.hasher::().root_bagging().domain(), ]; let output = Output { root: Keccak256::hash(&[&tags, &0u64.to_be_bytes()]).0, elements: vec![], proof: vec![], leaves: 0, }; let encoded = check_input(&output, &[], &[]); // The root commits `F` and `bagging`, so it verifies only under that pair. assert!(check::(&encoded, policy)); assert!(!check::(&encoded, other)); assert!(!check::(&encoded, policy)); assert!(!check::(&encoded, other)); // Malformed encodings and a nonzero inactive boundary fail even under `F` and `bagging`. let mut trailing = encoded.clone(); trailing.extend_from_slice(&[0; 32]); let mut overlapping = encoded.clone(); overlapping.copy_within(64..96, 96); for malformed in [&trailing, &overlapping] { assert!(!check::(malformed, policy)); } let policy = Policy { bagging, inactive_peaks: 1, }; assert!(!check::(&encoded, policy)); } #[test] fn cli_hash_selection() { for family in ["mmr", "mmb"] { for mode in ["generate-multi", "synthetic-multi"] { let mut outputs = Vec::new(); for hash in ["keccak256", "sha256"] { let encoded = Cli::try_parse_from([ "fuzz", "merkle", "--hash", hash, mode, "--family", family, "--leaf-count", "31", "--locations", "0,8,12,30", "--seed", "42", "--bagging", "backward", "--inactive-peaks", "2", ]) .unwrap() .command .execute() .unwrap(); let output = ::abi_decode_params_validate(&encoded).unwrap(); let input = MultiInput { root: output.root, leaves: output.leaves, locations: [0u64, 8, 12, 30].into_iter().map(Uint256::from).collect(), elements: output.elements, proof: output.proof, positions: output.positions, } .abi_encode_params(); let hex = const_hex::encode(input); for check_hash in ["keccak256", "sha256"] { let accepted = Cli::try_parse_from([ "fuzz", "merkle", "--hash", check_hash, "check-multi", "--family", family, "--abi", &hex, "--bagging", "backward", "--inactive-peaks", "2", ]) .unwrap() .command .execute() .unwrap(); assert_eq!(accepted, (hash == check_hash).abi_encode()); } outputs.push(encoded); } assert_ne!(outputs[0], outputs[1]); } } } #[test] fn cli_sparse_requires_named_fields() { assert!( Cli::try_parse_from([ "fuzz", "merkle", "--hash", "keccak256", "generate-multi", "mmb", "31", "0,30", "42" ]) .is_err() ); assert!( Cli::try_parse_from([ "fuzz", "merkle", "--hash", "keccak256", "generate-multi", "--family", "mmb", "--leaf-count", "31", "--locations", "0,30" ]) .is_err() ); } }