//! Shared synchronization logic for [crate::qmdb::current] databases. //! //! Contains the implementation of [crate::qmdb::sync::Database] for //! [Db](crate::qmdb::current::Db), covering every variant (ordered/unordered, //! fixed/variable, any snapshot index). //! //! The canonical root of a `current` database combines the ops root, grafted root, and optional //! pending and partial chunk digests into a single hash (see the [Root structure](super) section in //! the module documentation). The sync engine operates on the **ops root**, not the canonical root: //! it downloads operations and verifies each batch against the ops root using ops-tree range proofs //! (identical to `any` sync). Callers that verify current ops proofs directly should use //! [crate::qmdb::ProofExt::verify]. [crate::qmdb::current::proof::OpsRootWitness] can be used by //! callers that need to authenticate the synced ops root against a trusted canonical root; the sync //! engine does not perform this check itself. //! //! After all operations are synced, the bitmap and grafted tree are reconstructed deterministically //! from the operations. The canonical root is then computed from the ops root, the reconstructed //! grafted root, and any pending or partial chunk digests. //! //! The root of [`Database::target`] is the ops root used by the sync engine. //! //! For pruned databases (`range.start > 0`), grafted pinned nodes for the pruned region are read //! directly from the ops tree after it is built. This works because of the zero-chunk identity: for //! all-zero bitmap chunks (which all pruned chunks are), the grafted leaf equals the ops subtree //! root, making the grafted tree structurally identical to the ops tree at and above the grafting //! height. use crate::{ Context, index::{Factory as IndexFactory, Unordered as UnorderedIndex}, journal::{authenticated, contiguous::Mutable}, merkle::{Graftable, Location, storage::Storage as _}, qmdb::{ self, any::{ Db as AnyDb, operation::{Operation, update::Update}, }, bitmap::Shared, current::{db, grafting}, metrics::Metrics as AnyMetrics, sync::{Database, DatabaseConfig as Config, Hooks, Request, source}, }, translator::Translator, }; use commonware_codec::Codec; use commonware_cryptography::{DigestOf, Hasher}; use commonware_parallel::Strategy; use commonware_runtime::Spawner; use commonware_utils::{bitmap::Prunable as BitMap, range::NonEmptyRange}; use core::num::NonZeroUsize; use std::{num::NonZeroU64, sync::Arc}; #[cfg(test)] pub(crate) mod tests; impl Config for super::Config { type JournalConfig = J; fn journal_config(&self) -> Self::JournalConfig { self.log.clone() } } /// Shared helper to build a `current::Db` from sync components. /// /// This follows the same pattern as `any/sync/mod.rs::build_db` but additionally: /// * Builds the activity bitmap by replaying the operations log. /// * Extracts grafted pinned nodes from the ops tree (zero-chunk identity). /// * Builds the grafted tree from the bitmap and ops tree. /// * Computes and caches the canonical root. #[allow(clippy::too_many_arguments)] async fn build_db( context: E, merkle_config: authenticated::Config, log: J, translator: I::Translator, pinned_nodes: Option>, range: NonEmptyRange>, apply_batch_size: NonZeroU64, init_concurrency: NonZeroUsize, init_buffer: NonZeroUsize, cache_size: Option, metadata_partition: String, strategy: S, ) -> Result, qmdb::Error> where F: Graftable, E: Context + Spawner, U: Update, I: IndexFactory + crate::qmdb::SnapshotBuild, H: Hasher, J: Mutable> + 'static, S: Strategy, Operation: Codec, { // Build authenticated log. let index = I::new(context.child("index"), translator); let log = authenticated::Journal::::from_components( context.child("merkle"), &merkle_config, log, qmdb::hasher::(), range.start(), pinned_nodes.unwrap_or_default(), apply_batch_size, ) .await?; // Initialize bitmap with pruned chunks. // // Floor division is intentional: chunks entirely below range.start are pruned. // If range.start is not chunk-aligned, the partial leading chunk is reconstructed by // init_from_log, which pads the gap between `pruned_chunks * CHUNK_SIZE_BITS` and the // journal's inactivity floor with inactive (false) bits. let pruned_chunks = (*range.start() / BitMap::::CHUNK_SIZE_BITS) as usize; let bitmap = BitMap::::new_with_pruned_chunks(pruned_chunks) .map_err(|_| qmdb::Error::::DataCorrupted("pruned chunks overflow"))?; let bitmap = Arc::new(Shared::::new(bitmap)); // Build any::Db, handing it the pre-allocated bitmap. `init_from_log` populates the bitmap // during replay. let snapshot_context = context.child("any_snapshot"); let any_metrics = AnyMetrics::new(context.child("any")); let any: AnyDb = AnyDb::init_from_log( snapshot_context, index, log, Some(bitmap), init_concurrency, init_buffer, cache_size, any_metrics, ) .await?; // Fetch grafted pinned nodes from the ops tree. For each position the grafted family // needs at its pruning boundary, source the digest from the ops tree via the zero-chunk // identity: when the covered chunks are all zero (which pruned chunks always are), the // ops-family digest at the mapped position equals the grafted digest. // // Requires `range.start <=` target's [`Db::sync_boundary`](db::Db::sync_boundary): that // bound guarantees every required ops-tree node is born at `range.end`. let grafted_pinned_nodes = { let grafted_boundary = Location::::new(pruned_chunks as u64); let grafting_height = grafting::height::(); let mut pinned_nodes = Vec::new(); for grafted_pos in F::nodes_to_pin(grafted_boundary) { let ops_pos = grafting::grafted_to_ops_pos::(grafted_pos, grafting_height); let digest = any .log .get_node(ops_pos) .await? .ok_or(qmdb::Error::::DataCorrupted("missing ops pinned node"))?; pinned_nodes.push(digest); } pinned_nodes }; // Rebuild the grafted tree, graft roots, and ops root witness from the synced `any` state. The // resulting canonical root is deterministic because the engine authenticates the ops and the // bitmap is derived from them. let (grafted_tree, graft_roots, ops_root_witness) = db::rebuild_grafted_tree::( any.bitmap.as_ref(), &grafted_pinned_nodes, &any.log, any.inactivity_floor_loc, &strategy, ) .await?; // Initialize metadata store and construct the Db. let (metadata, _, _) = db::init_metadata::>(context.child("metadata"), &metadata_partition) .await?; let root = ops_root_witness.root::(&any.root()); let metrics = db::Metrics::new(context); let current_db = db::Db { any, grafted_tree: Arc::new(grafted_tree), graft_roots, metadata, strategy, root, ops_root_witness, metrics, #[cfg(test)] halt_before_prune_log: false, }; current_db.update_metrics(); Ok(current_db) } impl Database for db::Db where F: Graftable, E: Context + Spawner, C: crate::journal::authenticated::Backing< E, Item = Operation, Config = >::Config, > + crate::qmdb::sync::Journal> + 'static, I: IndexFactory + crate::qmdb::SnapshotBuild + UnorderedIndex>, H: Hasher, U: Update, S: Strategy, Operation: Codec, { type Family = F; type Context = E; type Op = Operation; type Journal = C; type Hasher = H; type Config = super::Config>::Config, S>; type Digest = H::Digest; async fn init( context: E, config: Self::Config, max_size: Option>, ) -> Result> { crate::qmdb::current::init_db(context, config, max_size).await } /// The root is the ops root (not the canonical root), since the sync engine verifies /// batches against the ops tree. fn target(&self) -> qmdb::sync::Target { qmdb::sync::Target { root: Self::ops_root(self), range: commonware_utils::non_empty_range!(self.sync_boundary(), self.bounds().end), } } async fn prune_to_target( self, target: &qmdb::sync::Target, ) -> Result> { self.prune(target.range.start()).await } } impl Hooks for db::Db where F: Graftable, E: Context + Spawner, C: crate::journal::authenticated::Backing< E, Item = Operation, Config = >::Config, > + crate::qmdb::sync::Journal> + 'static, I: IndexFactory + crate::qmdb::SnapshotBuild + UnorderedIndex>, H: Hasher, U: Update, S: Strategy, Operation: Codec, { async fn from_sync_result( context: E, config: ::Config, log: C, pinned_nodes: Option>, range: NonEmptyRange>, apply_batch_size: NonZeroU64, ) -> Result> { let strategy = config.merkle.strategy.clone(); build_db::( context, config.merkle, log, config.translator, pinned_nodes, range, apply_batch_size, config.init_concurrency, config.init_buffer, config.init_cache, config.grafted_metadata_partition, strategy, ) .await } async fn persist_sync_result(mut self) -> Result> { // The grafted metadata must never leave a chunk unpruned when the frontier cannot serve its // graft root. A boundary that moves forward writes the metadata first, as Db::prune does. // One that moves back writes the frontier first, which restores access to the earlier roots // before the metadata needs them. if self.any.log.merkle.bounds().start < self.any.log.frontier.boundary().0 { self.any.log = self.any.log.persist_boundary().await?; self = self.sync_metadata().await?; } else { self = self.sync_metadata().await?; self.any.log = self.any.log.persist_boundary().await?; } Ok(self) } async fn discard_local_state( context: E, config: &::Config, ) -> Result<(), qmdb::Error> { authenticated::discard_frontier::(context.child("merkle"), &config.merkle) .await?; // Grafted metadata records chunks pruned from operations the log held, so it is // discarded with the frontier of those operations. let (mut metadata, pruned_chunks, _) = db::init_metadata::>( context.child("metadata"), &config.grafted_metadata_partition, ) .await?; if pruned_chunks > 0 { metadata.clear(); metadata.sync().await?; } Ok(()) } async fn local_pinned_nodes( context: E, config: &::Config, target: &qmdb::sync::Target, journal: &C, ) -> Result>, qmdb::Error> { qmdb::sync::local_pinned_nodes::(context, &config.merkle, target, journal).await } } /// A `current` database serves proofs from the `any` database it wraps. The sync engine /// operates on the ops root, which is `any`'s root. impl crate::qmdb::sync::Source for db::Db where F: Graftable, E: Context, C: Mutable>, I: UnorderedIndex>, H: Hasher, U: Update, S: Strategy, Operation: Codec, { type Family = F; type Digest = H::Digest; type Op = Operation; type Error = qmdb::Error; async fn serve(&self, request: Request) -> source::Result { self.any.serve(request).await } }