//! Segmented journal for oversized values. //! //! This module combines [super::fixed::Journal] with [super::glob::Glob] to handle //! entries that reference variable-length "oversized" values. It provides coordinated //! operations and built-in crash recovery. //! //! # Architecture //! //! ```text //! +-------------------+ +-------------------+ //! | Fixed Journal | | Glob (Values) | //! | (Index Entries) | | | //! +-------------------+ +-------------------+ //! | entry_0 | --> | value_0 | //! | entry_1 | --> | value_1 | //! | ... | | ... | //! +-------------------+ +-------------------+ //! ``` //! //! Each index entry contains `(value_offset, value_size)` pointing to its value in glob. //! //! # Crash Recovery //! //! Values are appended to the glob before their index entries, so an unclean shutdown can leave //! an index entry without durable value bytes, value bytes without an index entry, or a value //! section without an index section. The index is the authority. Recovery durably shortens it //! before releasing the value bytes it no longer references, so later appends cannot expose a //! dropped entry or its stale bytes. Each index section ends up paired with a value section that //! stops at its last retained entry, and value sections without an index section are removed. //! //! The owner selects one of two recovery contracts at initialization. //! //! ## Checkpoint restore //! //! [Oversized::init_with_checkpoint] restores a `(section, index size)` checkpoint that the owner //! published only after a completed [Oversized::sync] covered it. Each section below the //! checkpoint is adopted at its validated terminal boundary without reading values, the //! checkpoint section is durably truncated to the committed size, and everything after it is //! removed unread. A missing or damaged boundary the checkpoint covers fails init, and other //! committed damage surfaces lazily as read errors. //! //! Restoring the empty checkpoint `(0, 0)` on empty storage opens a fresh journal. A restored //! journal keeps no markers, so its owner is responsible for publishing each later checkpoint. //! //! ## Marker-tracked recovery //! //! [Oversized::init_with_markers] persists a per-section committed item count. Entries below the //! marker are adopted once their index/value boundary is proven, entries above it are //! value-verified in order, and the first invalid value truncates the section's remainder. Markers //! trail proven syncs, publishing once a section is durable and idle (or on an empty flush). use super::{ fixed::{Config as FixedConfig, Journal as FixedJournal, Recovery as FixedRecovery}, glob::{Config as GlobConfig, Glob, Recovery as GlobRecovery}, }; use crate::{ Context, SyncCompletion, journal::{Error, durability::Barrier}, metadata::{Config as MetadataConfig, Metadata}, }; use commonware_codec::{Codec, CodecFixedShared, CodecShared}; use commonware_runtime::{Handle, ReadOptions}; use commonware_utils::sequence::U64 as SectionKey; use futures::{FutureExt as _, future::try_join}; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, num::NonZeroUsize, }; use tracing::warn; /// Trait for index entries that reference oversized values in glob storage. /// /// Implementations must provide access to the value location for crash recovery validation, /// and a way to set the location when appending. pub trait Record: CodecFixedShared + Clone { /// Returns `(value_offset, value_size)` for crash recovery validation. fn value_location(&self) -> (u64, u32); /// Returns a new entry with the value location set. /// /// Called during `append` after the value is written to glob storage. fn with_location(self, offset: u64, size: u32) -> Self; } /// Configuration for oversized journal. #[derive(Clone)] pub struct Config { /// Partition for the fixed index journal. pub index_partition: String, /// Partition for the glob value storage. pub value_partition: String, /// Page cache for index journal caching. pub index_page_cache: commonware_runtime::buffer::paged::CacheRef, /// Write buffer size for the index journal. pub index_write_buffer: NonZeroUsize, /// Write buffer size for the value journal. pub value_write_buffer: NonZeroUsize, /// Buffer size for sequential index recovery. pub replay_buffer: NonZeroUsize, /// Optional zstd compression level for stored values. /// /// Keep the choice between `None` and `Some(_)` fixed while stored values are retained. /// Only the compression level may change between initializations when compression is enabled. pub compression: Option, /// Codec configuration for values. pub codec_config: C, } /// Durable recovery state for a journal that validates every uncommitted value during replay. struct Tracking { /// Per-section committed item counts, including staged updates awaiting marker persistence. metadata: Metadata, /// Completion of the marker generation currently being persisted. marker_sync_pending: Option, /// Joint index/value durability proofs whose item counts are not yet fully published as /// markers. barriers: BTreeMap, } impl Tracking { /// Stage a changed marker while preserving absence as the canonical zero boundary. fn stage_marker(&mut self, section: u64, floor: u64) -> bool { let key = SectionKey::new(section); match self.metadata.get(&key) { None if floor == 0 => false, Some(stored) if *stored == floor => false, _ => { self.metadata.put(key, floor); true } } } /// Return the section's barrier, seeding a replacement at its staged floor. /// /// A staged floor never exceeds durably synced data, so it is the newest boundary a /// replacement barrier may claim without observing a completed sync. fn barrier(&mut self, section: u64) -> &mut Barrier { self.barriers.entry(section).or_insert_with(|| { let floor = self .metadata .get(&SectionKey::new(section)) .copied() .unwrap_or(0); Barrier::new(floor) }) } /// Observe an in-flight marker without blocking and discard proofs it published. /// /// Returns whether a marker generation is still in flight. fn observe_marker_sync(&mut self) -> Result { let Some(completion) = self.marker_sync_pending.as_mut() else { return Ok(false); }; let Some(result) = completion.now_or_never() else { return Ok(true); }; result.map_err(|err| Error::Metadata(crate::metadata::Error::Runtime(err)))?; self.marker_sync_pending = None; // Retire only barriers whose proof is fully published. A barrier still awaiting a // sync outcome protects a boundary beyond its marker and must survive to observe it. let metadata = &self.metadata; self.barriers.retain(|section, barrier| { let published = metadata .get(&SectionKey::new(*section)) .copied() .unwrap_or(0); barrier.boundary() > published || !barrier.settled() }); Ok(false) } } /// State for the one marker-aware replay performed while opening a tracked journal. struct Validation { /// Section whose cached `floor` and `truncated` state is active. current_section: Option, /// Committed item count from the durable marker; replay checks value checksums only at or /// above it. floor: u64, /// Whether this section reached its first invalid value, so later index entries are skipped. truncated: bool, /// First invalid item position per section, converted to an index-byte truncation after /// replay releases the index journal. truncations: Vec<(u64, u64)>, /// Whether replay yielded an error that makes the journal unavailable. failed: bool, } impl Validation { const fn new() -> Self { Self { current_section: None, floor: 0, truncated: false, truncations: Vec::new(), failed: false, } } } /// Segmented journal for entries with oversized values. /// /// Combines a fixed-size index journal with glob storage for variable-length values. /// Provides coordinated operations and crash recovery. /// /// Mutating functions consume the journal and return it only on success: an error (or a dropped /// future) destroys the handle. Mutations on pruned sections fail with /// [Error::AlreadyPrunedToSection]. Check [Oversized::pruned] first to keep the handle. pub struct Oversized { index: FixedJournal, values: Glob, tracking: Option>, } /// Owns both journals while initialization may still discard unreferenced data. struct Pending { /// Unpublished index whose per-section byte extents are reconciled with value storage. index: FixedJournal, /// Unpublished values; bytes owned by a discarded index suffix are released only after its /// truncation is durable. values: GlobRecovery, } impl std::fmt::Debug for Oversized { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Oversized") .field("oldest_section", &self.oldest_section()) .field("newest_section", &self.newest_section()) .finish_non_exhaustive() } } impl From> for Oversized { /// Publish both journals after checkpoint restore. A restored journal keeps no markers. fn from(recovery: Pending) -> Self { Self { index: recovery.index, values: recovery.values.finish(), tracking: None, } } } impl Pending { /// Split `cfg` into the configurations of the index and value journals. fn configs(cfg: Config) -> (FixedConfig, GlobConfig) { let index_cfg = FixedConfig { partition: cfg.index_partition, page_cache: cfg.index_page_cache, write_buffer: cfg.index_write_buffer, replay_buffer: cfg.replay_buffer, }; let value_cfg = GlobConfig { partition: cfg.value_partition, compression: cfg.compression, codec_config: cfg.codec_config, write_buffer: cfg.value_write_buffer, }; (index_cfg, value_cfg) } /// Prove each per-section validated prefix in both journals and return the recoveries that /// repair the suffixes above them. The index recovery has modified nothing: its replay /// starts only after every floor's value extent is verified. async fn preflight_floors( context: E, cfg: Config, floors: &BTreeMap, read_options: ReadOptions, ) -> Result<(FixedRecovery, GlobRecovery), Error> { let (index_cfg, value_cfg) = Self::configs(cfg); let index = FixedJournal::preflight_floors(context.child("index"), index_cfg, floors, read_options) .await?; let values = GlobRecovery::init(context.child("values"), value_cfg).await?; Self::validate_value_floors(&values, &index)?; Ok((index, values)) } /// Open both journals at an explicit checkpoint, restoring `section` to `index_size` index /// bytes and removing every later section unread. async fn restore( context: E, cfg: Config, section: u64, index_size: u64, ) -> Result { let (index_cfg, value_cfg) = Self::configs(cfg); let preflight = FixedJournal::preflight_restore(context.child("index"), index_cfg, section, index_size) .await?; let values = GlobRecovery::init_bounded(context.child("values"), value_cfg, section).await?; let value_size = Self::validate_restore_values(&values, &preflight, section)?; let index = preflight.finish().await?; // The index truncation is already durable. Release its unreferenced values only after // that proof, preserving the index-first crash-recovery order. let values = values.truncate(section, value_size).await?; Ok(Self { index, values: values.sync(section).await?, }) } /// Return the value boundary owned by an optional terminal index entry. fn boundary_value_end(section: u64, entry: &Option) -> Result { let Some(entry) = entry else { return Ok(0); }; let (offset, size) = entry.value_location(); offset.checked_add(u64::from(size)).ok_or_else(|| { Error::Corruption(format!( "section {section} has an overflowing terminal value range" )) }) } /// Verify every floor's terminal value extent before repair can mutate either journal. fn validate_value_floors( values: &GlobRecovery, preflight: &FixedRecovery, ) -> Result<(), Error> { for (§ion, entry) in preflight.boundaries() { let required = Self::boundary_value_end(section, entry)?; if required == 0 { continue; } let retained = values.size(section)?; if retained < required { return Err(Error::Corruption(format!( "section {section} retains {retained} value bytes, below the validation \ floor of {required}" ))); } } Ok(()) } /// Verify checkpoint-covered value extents against preflighted index boundaries, returning /// the checkpoint section's terminal value end. fn validate_restore_values( values: &GlobRecovery, preflight: &FixedRecovery, section: u64, ) -> Result { // Every earlier section is immutable under the checkpoint, so its terminal index entry // must end exactly at the retained value length. for (&candidate, entry) in preflight.boundaries().range(..section) { let required = Self::boundary_value_end(candidate, entry)?; let retained = values.size(candidate)?; if retained != required { return Err(Error::Corruption(format!( "section {candidate} index ends at value byte {required}, but its glob size is {retained}" ))); } } // A value-only section between the oldest retained index section and the checkpoint // proves that covered index data was lost. Value-only sections below the oldest index // section are what a prune interrupted between its index and value removals leaves, // and the restore removes them. let oldest_index = *preflight .boundaries() .keys() .next() .expect("preflight records the checkpoint section"); if let Some(orphan) = values.sections().find(|candidate| { (oldest_index..section).contains(candidate) && !preflight.boundaries().contains_key(candidate) }) { return Err(Error::Corruption(format!( "section {orphan} has values but no index" ))); } // The current section may retain a suffix, but it must cover its committed terminal value. let required = Self::boundary_value_end(section, &preflight.boundaries()[§ion])?; if required > 0 { let retained = values.size(section)?; if retained < required { return Err(Error::Corruption(format!( "section {section} retains {retained} of {required} committed value bytes" ))); } } Ok(required) } /// Remove any value sections that don't have corresponding index sections. /// /// This can happen if a crash occurs after writing to values but before /// writing to index for a new section. Since sections don't have to be /// contiguous, we compare the actual sets of sections rather than just /// comparing the newest section numbers. async fn cleanup_orphan_value_sections(mut self) -> Result { // Collect index sections into a set for O(1) lookup let index_sections: HashSet = self.index.sections().collect(); // Find value sections that don't exist in index let orphan_sections: Vec = self .values .sections() .filter(|s| !index_sections.contains(s)) .collect(); // Remove each orphan section for section in orphan_sections { warn!(section, "removing orphan value section"); self.values = self.values.remove_section(section).await?; } Ok(self) } /// Truncate each section's values to the end of its last index entry, then remove value /// sections that have no index section. Values extend past the last entry when replay repairs /// index pages or validation truncates a section at its first invalid value. async fn align_values_to_index(mut self) -> Result { let sections = self.index.sections().collect::>(); let mut value_truncations = BTreeMap::new(); for section in sections { let target = Self::boundary_value_end(section, &self.index.last(section).await?)?; let retained = self.values.size(section)?; if retained < target { return Err(Error::Corruption(format!( "section {section} retains {retained} of {target} indexed value bytes" ))); } if retained > target { value_truncations.insert(section, target); } } self.values = self.values.truncate_sections(&value_truncations).await?; self.cleanup_orphan_value_sections().await } /// Reconcile durable markers with the retained state after tracked startup recovery. /// /// Markers rise to each section's replayed length without syncing the data journals. The /// retained entries were readable when the run began, which the [commonware_runtime::Storage] /// durability contract makes crash-durable, and recovery truncations are durable before this /// runs. Returns the published tracked journal. async fn reconcile_markers( self, mut tracking: Tracking, ) -> Result, Error> { let mut dirty = false; for section in self.index.sections() { let items = self.index.section_len(section)?; dirty |= tracking.stage_marker(section, items); } if dirty { let marker; (tracking.metadata, marker) = tracking.metadata.start_sync().await?; tracking.marker_sync_pending = Some(marker.boxed().shared()); } Ok(Oversized { index: self.index, values: self.values.finish(), tracking: Some(tracking), }) } } impl Oversized { /// Initialize with crash recovery restoring a durable `(section, index size)` checkpoint. /// /// Recovery keeps exactly the checkpointed state. Each section below `section` is adopted at /// its validated terminal boundary, `section` is truncated to `index size`, and everything /// after it is removed without being opened. A prune interrupted between its index and value /// removals can leave value sections older than the oldest retained index section, and /// recovery removes them too. /// /// A missing or damaged boundary the checkpoint covers fails init with [Error::Corruption], /// and interior damage below a boundary surfaces lazily as read errors. Every blob name in /// the index and value partitions must still decode as a section number, including those /// after the checkpoint. /// /// Callers must only provide a checkpoint that was durably synced before it was published. The /// empty checkpoint `(0, 0)` opens empty storage without creating or writing any blob. pub async fn init_with_checkpoint( context: E, cfg: Config, checkpoint: (u64, u64), ) -> Result { let (section, index_size) = checkpoint; let journal = Pending::restore(context, cfg, section, index_size).await?; Ok(journal.cleanup_orphan_value_sections().await?.into()) } /// Initialize using durable per-section validation markers and return the [Recovery] to drain. /// /// The caller must drain it and call [Recovery::finish]. Entries below each durable marker are /// retained after their cross-journal boundary is proven. Entries above it are value-validated /// in order and the first invalid entry truncates its section. pub async fn init_with_markers( context: E, cfg: Config, marker_partition: String, read_options: ReadOptions, ) -> Result, Error> { // Open the commit markers before the data they constrain. let metadata: Metadata = Metadata::init( context.child("metadata"), MetadataConfig { partition: marker_partition, codec_config: (), }, ) .await?; let floors = metadata .keys() .map(|key| { ( u64::from(key), *metadata.get(key).expect("metadata key must have a value"), ) }) .collect::>(); // Every advertised prefix is proven before ordinary suffix repair may mutate either // journal. An empty sidecar commits no prefix, so replay validates every value. let (index, values) = Pending::preflight_floors(context, cfg, &floors, read_options).await?; // Keep the durable floors attached while replay validates each uncommitted suffix. Ok(Recovery { index, values, validation: Validation::new(), tracking: Tracking { metadata, marker_sync_pending: None, barriers: BTreeMap::new(), }, in_progress: false, interrupted: false, }) } /// Append entry + value. /// /// Writes value to glob first, then writes index entry with the value location. /// /// Returns `(self, position, offset, size)` where: /// - `position`: Position in the index journal /// - `offset`: Byte offset in glob /// - `size`: Size of value in glob (including checksum) pub async fn append( mut self, section: u64, entry: I, value: &V, ) -> Result<(Self, u64, u64, u32), Error> { // Write value first (glob). This will typically write to an in-memory // buffer and return quickly (only blocks when the buffer is full). let (offset, size); (self.values, offset, size) = self.values.append(section, value).await?; // Update entry with actual location and write to index let entry_with_location = entry.with_location(offset, size); let position; (self.index, position) = self.index.append(section, &entry_with_location).await?; // Track this section so a later sync can prove and publish its new length. A fresh // barrier claims only the staged floor, never the unproven pre-append prefix. if let Some(tracking) = &mut self.tracking { tracking.barrier(section); } Ok((self, position, offset, size)) } /// Get entry at position (index entry only, not value). pub async fn get(&self, section: u64, position: u64) -> Result { self.index.get(section, position).await } /// Get the last entry for a section, if any. /// /// Returns `Ok(None)` if the section is empty. /// /// # Errors /// /// - [Error::AlreadyPrunedToSection] if the section has been pruned. /// - [Error::SectionOutOfRange] if the section doesn't exist. pub async fn last(&self, section: u64) -> Result, Error> { self.index.last(section).await } /// Get value using offset/size from entry. /// /// The offset should be the byte offset from `append()` or from the entry's `value_location()`. pub async fn get_value(&self, section: u64, offset: u64, size: u32) -> Result { self.values.get(section, offset, size).await } /// Start a joint data sync and publish only previously completed tracked boundaries. /// /// The returned handle covers only the requested data syncs. Marker generations trail data /// durability, and a marker failure surfaces as [Error::Metadata] when a later request /// observes it. pub(crate) async fn start_sync_tracked( mut self, sections: &BTreeSet, active: &BTreeSet, ) -> Result<(Self, Handle<()>), Error> { let mut tracking = self .tracking .take() .expect("tracked sync preserves its recovery state"); let marker_pending = tracking.observe_marker_sync()?; let lengths = sections .iter() .map(|§ion| Ok((section, self.index.section_len(section)?))) .collect::, Error>>()?; let ((index, index_handle), (values, values_handle)) = try_join( self.index.start_sync(sections), self.values.start_sync(sections), ) .await?; self.index = index; self.values = values; let completion: SyncCompletion = async move { try_join(index_handle, values_handle).await.map(|_| ()) } .boxed() .shared(); // Bind every selected target to the new joint completion. The recorded length becomes // publishable only once this completion is observed to have succeeded. for (section, length) in lengths { tracking.barrier(section).record(length, completion.clone()); } // Do not mutate Metadata while its prior generation is in flight. Completed barriers stay // as debt and are published when their section is no longer active, or on an empty flush. if !marker_pending { let publish = tracking .barriers .iter_mut() .filter(|(section, _)| !active.contains(section)) .map(|(§ion, barrier)| (section, barrier.boundary())) .collect::>(); let mut metadata_dirty = false; for (section, floor) in publish { metadata_dirty |= tracking.stage_marker(section, floor); } if metadata_dirty { let handle; (tracking.metadata, handle) = tracking.metadata.start_sync().await?; tracking.marker_sync_pending = Some(handle.boxed().shared()); } } self.tracking = Some(tracking); Ok((self, Handle::from_future(completion))) } /// Block until the selected data syncs are durable, surfacing any marker failure the /// completed generation already exposed. pub(crate) async fn sync_tracked( self, sections: &BTreeSet, active: &BTreeSet, ) -> Result { let (mut journal, handle) = self.start_sync_tracked(sections, active).await?; handle.await?; journal .tracking .as_mut() .expect("tracking mode is preserved") .observe_marker_sync()?; Ok(journal) } /// Sync both journals for the given `sections`. pub async fn sync(mut self, sections: impl crate::Sections) -> Result { if self.tracking.is_some() { let sections = sections.sections().collect::>(); return self.sync_tracked(§ions, §ions).await; } let sections = sections.sections().collect::>(); (self.index, self.values) = try_join(self.index.sync(§ions), self.values.sync(§ions)).await?; Ok(self) } /// Start syncing both journals for the given `sections`. /// /// The returned handle completes once both journals' syncs complete, failing with the first /// error encountered. An error reported by the returned [Handle] is fatal to the journal: /// the caller must stop using the returned journal. pub async fn start_sync( mut self, sections: impl crate::Sections, ) -> Result<(Self, Handle<()>), Error> { if self.tracking.is_some() { let sections = sections.sections().collect::>(); return self.start_sync_tracked(§ions, §ions).await; } let sections = sections.sections().collect::>(); let ((index, index_handle), (values, values_handle)) = try_join( self.index.start_sync(§ions), self.values.start_sync(§ions), ) .await?; self.index = index; self.values = values; Ok(( self, Handle::from_future( async move { try_join(index_handle, values_handle).await.map(|_| ()) }, ), )) } /// Sync all sections. pub async fn sync_all(mut self) -> Result { if self.tracking.is_some() { let sections = self.index.sections().collect::>(); return self.sync(sections).await; } (self.index, self.values) = try_join(self.index.sync_all(), self.values.sync_all()).await?; Ok(self) } /// Prune both journals. Returns true if any sections were pruned. /// /// Prunes index first, then glob. This order ensures crash safety: /// - If crash after index prune but before glob: orphan data in glob (acceptable) /// - If crash before index prune: no change, retry works pub async fn prune(mut self, min: u64) -> Result<(Self, bool), Error> { // Remove and durably sync tracked floors before their section names can be deleted and // later reused. if let Some(mut tracking) = self.tracking.take() { let mut removed = false; tracking.metadata.retain(|key, _| { let keep = u64::from(key) >= min; removed |= !keep; keep }); if removed { tracking.metadata = tracking.metadata.sync().await?; tracking.marker_sync_pending = None; } tracking.barriers = tracking.barriers.split_off(&min); self.tracking = Some(tracking); } let index_pruned; (self.index, index_pruned) = self.index.prune(min).await?; let value_pruned; (self.values, value_pruned) = self.values.prune(min).await?; Ok((self, index_pruned || value_pruned)) } /// Get index size for checkpoint. /// /// The value size can be derived from the last entry's location when needed. pub fn size(&self, section: u64) -> Result { self.index.size(section) } /// Get the value size for a section, derived from the last entry's location. pub async fn value_size(&self, section: u64) -> Result { match self.index.last(section).await { Ok(Some(entry)) => { let (offset, size) = entry.value_location(); offset .checked_add(u64::from(size)) .ok_or(Error::OffsetOverflow) } Ok(None) | Err(Error::SectionOutOfRange(_)) => Ok(0), Err(e) => Err(e), } } /// Returns true when `section` is below the prune floor. /// /// The floor only tracks prunes from the current execution and resets at init, so a /// section pruned in a previous execution reports false. pub fn pruned(&self, section: u64) -> bool { self.index.pruned(section) } /// Returns the oldest section number, if any exist. pub fn oldest_section(&self) -> Option { self.index.oldest_section() } /// Returns the newest section number, if any exist. pub fn newest_section(&self) -> Option { self.index.newest_section() } /// Destroy all underlying storage. pub async fn destroy(mut self) -> Result<(), Error> { // Remove tracked floors first so an interrupted destroy can only force conservative // recovery of any pair data left behind. if let Some(tracking) = self.tracking.take() { tracking.metadata.destroy().await?; } try_join(self.index.destroy(), self.values.destroy()) .await .map(|_| ()) } } /// Validating startup recovery returned by [Oversized::init_with_markers]. /// /// Yields `(section, position, entry)` in order, verifying each value at or above its section's /// durable marker until the first invalid one. Dropping it before [Recovery::finish] destroys the /// handle, and the owner must initialize again. pub struct Recovery { /// Index recovery, which modifies nothing until its first entry is requested. index: FixedRecovery, /// Value recovery owner that validates entries at or above each durable floor until the /// first invalid value. values: GlobRecovery, /// Cursor and deferred truncation state for the replay. validation: Validation, /// Durable markers whose floors bound value validation. tracking: Tracking, in_progress: bool, interrupted: bool, } impl Recovery { /// Returns the next `(section, position, entry)`, or `None` once every section is exhausted. /// /// The first call opens the index sections, which may discard a torn tail page. An error /// opening them ends the replay. /// /// An index error ends the section that produced it, and iteration continues with the next /// section. A value-verification error is returned without ending its section, and finishing /// the recovery then fails with [Error::ReplayFailed]. Errors while mutating storage to repair /// a section end the replay. Dropping a pending call also ends it. The next call returns /// [Error::ReplayInterrupted] and [Recovery::finish] fails. pub async fn next(&mut self) -> Option> { if self.interrupted { return None; } if self.in_progress { self.in_progress = false; self.interrupted = true; return Some(Err(Error::ReplayInterrupted)); } // Leave this set if the future is dropped at any await in the replay. self.in_progress = true; let result = self.next_inner().await; self.in_progress = false; result } async fn next_inner(&mut self) -> Option> { loop { let result = self.index.next().await?; let (section, position, entry) = match result { Ok(entry) => entry, Err(err) => return Some(Err(err)), }; let validation = &mut self.validation; // Each section validates forward from its durable floor. Once one value fails, later // entries in that section are outside the retained prefix and are not yielded. if validation.current_section != Some(section) { validation.current_section = Some(section); validation.floor = self .tracking .metadata .get(&SectionKey::new(section)) .copied() .unwrap_or(0); validation.truncated = false; } if validation.truncated { continue; } if position < validation.floor { return Some(Ok((section, position, entry))); } let (offset, size) = entry.value_location(); match self.values.verify(section, offset, size).await { Ok(true) => return Some(Ok((section, position, entry))), Ok(false) => { warn!(section, position, "invalid value detected: truncating"); validation.truncations.push((section, position)); validation.truncated = true; } Err(err) => { validation.failed = true; return Some(Err(err)); } } } } /// Returns the journal. /// /// Fails with [Error::ReplayFailed] when the recovery was not fully drained, yielded an /// error, or was interrupted. Otherwise each section is durably truncated at its first /// invalid entry, values no retained entry references are released, and the retained /// lengths are staged as the next markers. pub async fn finish(self) -> Result, Error> { if self.in_progress || self.interrupted || self.validation.failed { return Err(Error::ReplayFailed); } let mut journal = Pending { index: self.index.finish().await?, values: self.values, }; // Apply each section's first invalid position to the index only after replay releases the // index journal. Each truncation is durable before the alignment truncates the values past // it, and the exact retained lengths are then published as the next marker generation. let chunk_size = FixedJournal::::CHUNK_SIZE as u64; for (section, items) in self.validation.truncations { let index_size = items.checked_mul(chunk_size).ok_or(Error::OffsetOverflow)?; journal.index = journal .index .truncate_pending_section(section, index_size) .await?; } journal .align_values_to_index() .await? .reconcile_markers(self.tracking) .await } } #[cfg(test)] mod tests { use super::*; use commonware_codec::{Buf, FixedSize, Read, ReadExt, Write}; use commonware_cryptography::Crc32; use commonware_macros::test_traced; use commonware_runtime::{ Blob as _, BufMut, Clock as _, Error as RError, Runner, Storage as _, Supervisor as _, WriteOptions, buffer::paged::{CacheRef, corrupt_page}, deterministic, mocks::{ DeferredSync, DelayedSyncContext, MutationFaultContext, MutationFaults, PendingSyncs, ReadGateContext, SyncFaultContext, drive_pending_syncs, next_pending_sync, }, }; use commonware_utils::{NZU16, NZUsize, Widen, probability}; use std::time::Duration; /// Convert offset + size to byte end position (for truncation tests). fn byte_end(offset: u64, size: u32) -> u64 { offset + u64::from(size) } /// Leave a partial terminal index page that paged recovery would repair on open. async fn tear_index_tail( context: &impl commonware_runtime::Storage, cfg: &Config<()>, section: u64, ) { let (blob, len) = context .open(&cfg.index_partition, §ion.to_be_bytes()) .await .expect("failed to open index blob"); blob.resize(len - 1).await.expect("failed to tear tail"); blob.sync().await.expect("failed to sync torn tail"); } /// Test index entry that stores a u64 id and references a value. #[derive(Debug, Clone, PartialEq)] struct TestEntry { id: u64, value_offset: u64, value_size: u32, } impl TestEntry { fn new(id: u64, value_offset: u64, value_size: u32) -> Self { Self { id, value_offset, value_size, } } } impl Write for TestEntry { fn write(&self, buf: &mut impl BufMut) { self.id.write(buf); self.value_offset.write(buf); self.value_size.write(buf); } } impl Read for TestEntry { type Cfg = (); fn read_cfg(buf: &mut impl Buf, _: &Self::Cfg) -> Result { let id = u64::read(buf)?; let value_offset = u64::read(buf)?; let value_size = u32::read(buf)?; Ok(Self { id, value_offset, value_size, }) } } impl FixedSize for TestEntry { const SIZE: usize = u64::SIZE + u64::SIZE + u32::SIZE; } /// Encoded size of one index entry. const ENTRY_SIZE: u64 = TestEntry::SIZE as u64; impl Record for TestEntry { fn value_location(&self) -> (u64, u32) { (self.value_offset, self.value_size) } fn with_location(mut self, offset: u64, size: u32) -> Self { self.value_offset = offset; self.value_size = size; self } } /// Aligning value tails to the index truncates independent sections concurrently, so each /// section's durable value truncation reaches its sync before any of them completes. #[test] fn test_oversized_orphan_value_alignment_sync_concurrency() { deterministic::Runner::default().start(|context| async move { let cfg = entry_cfg(); // Seed one durable entry per section. let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("seed"), cfg.clone(), (0, 0)) .await .unwrap(); for section in 1..=2 { (journal, _, _, _) = journal .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .unwrap(); } journal = journal.sync_all().await.unwrap(); // The value sync materializes a crash image with durable value tails and // index additions still confined to the larger index write buffers. for section in 1..=2 { (journal, _, _, _) = journal .append( section, TestEntry::new(100 + section, 0, 0), &[(100 + section) as u8; 16], ) .await .unwrap(); } journal.values = journal.values.sync_all().await.unwrap(); drop(journal); // Reopen and replay the index, leaving each section's value tail longer than its // last retained entry. let outer = PendingSyncs::default(); let inner = PendingSyncs::default(); let delayed = DelayedSyncContext { inner: DelayedSyncContext { inner: context.child("open"), pending: inner.clone(), }, pending: outer.clone(), }; let (mut index, values) = Pending::<_, TestEntry, TestValue>::preflight_floors( delayed, cfg.clone(), &BTreeMap::new(), ReadOptions::default(), ) .await .unwrap(); while let Some(item) = index.next().await { item.unwrap(); } let recovery = Pending { index: index.finish().await.unwrap(), values, }; for section in 1..=2 { assert_eq!(recovery.index.size(section).unwrap(), ENTRY_SIZE); let entry = recovery.index.get(section, 0).await.unwrap(); let (offset, size) = entry.value_location(); assert!(recovery.values.size(section).unwrap() > offset + u64::from(size)); } // The outer gate holds the first value sync. A second value sync // passes that consumed gate and waits at the inner gate. outer.arm(); inner.arm(); let DeferredSync { blocked: first_blocked, release: first_release, } = next_pending_sync(&outer); let DeferredSync { blocked: second_blocked, release: second_release, } = next_pending_sync(&inner); let mut aligning = Box::pin(recovery.align_values_to_index()); commonware_macros::select! { _ = aligning.as_mut() => { panic!("recovery completed before both value syncs were released"); }, _ = async { first_blocked.await.unwrap(); second_blocked.await.unwrap(); } => {}, _ = context.sleep(Duration::from_secs(1)) => { panic!("independent value truncations did not reach both sync gates"); }, } assert_eq!(outer.calls(), 2); assert_eq!(inner.calls(), 1); second_release.send(Ok(())).unwrap(); first_release.send(Ok(())).unwrap(); let journal: Oversized<_, TestEntry, TestValue> = aligning.await.unwrap().into(); assert_eq!(inner.calls(), 2); // Each section keeps its entry, and its value tail ends at that entry. for section in 1..=2 { assert_eq!(journal.size(section).unwrap(), ENTRY_SIZE); let entry = journal.get(section, 0).await.unwrap(); assert_eq!(entry.id, section); let (offset, size) = entry.value_location(); assert_eq!( journal.get_value(section, offset, size).await.unwrap(), [section as u8; 16], ); assert_eq!( journal.values.size(section).unwrap(), offset + u64::from(size), ); } journal.destroy().await.unwrap(); }); } fn test_cfg() -> Config<()> { Config { index_partition: "test-index".into(), value_partition: "test-values".into(), index_page_cache: CacheRef::new(NZU16!(64), NZUsize!(8)), index_write_buffer: NZUsize!(1024), value_write_buffer: NZUsize!(1024), replay_buffer: NZUsize!(4096), compression: None, codec_config: (), } } /// Test configuration sized so each index page holds exactly one entry. fn entry_cfg() -> Config<()> { let mut cfg = test_cfg(); cfg.index_page_cache = CacheRef::new(NZU16!(TestEntry::SIZE as u16), NZUsize!(8)); cfg } /// Simple test value type with unit config. type TestValue = [u8; 16]; /// Marker partition of the journals [open_tracked] opens. const MARKERS: &str = "test-markers"; /// Open a marker-tracked journal and drain its validation replay. async fn open_tracked( context: E, cfg: Config<()>, ) -> Result, Error> { let mut replay = Oversized::init_with_markers(context, cfg, MARKERS.into(), ReadOptions::default()) .await?; while let Some(item) = replay.next().await { item?; } replay.finish().await } /// A tracked reopen whose markers already match every replayed section writes no marker /// generation. #[test] fn test_tracked_reopen_preserves_matching_markers() { deterministic::Runner::default().start(|context| async move { // Seed a tracked journal and publish a marker covering both of its entries. let cfg = test_cfg(); let mut journal = open_tracked(context.child("seed"), cfg.clone()) .await .unwrap(); for id in 0..2 { (journal, _, _, _) = journal .append(0, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } journal = journal.sync(0).await.unwrap(); let mut tracking = journal.tracking.take().unwrap(); assert!(tracking.stage_marker(0, 2)); tracking.metadata = tracking.metadata.sync().await.unwrap(); drop(tracking); drop(journal); // Reopening replays both entries without rewriting the marker generation. let (context, recordings) = commonware_runtime::mocks::RecordingContext::new(context); let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("reopen"), cfg, MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); let mut ids = Vec::new(); while let Some(item) = replay.next().await { ids.push(item.unwrap().2.id); } assert_eq!(ids, vec![0, 1]); drop(replay.finish().await.unwrap()); assert!( recordings.snapshot().writes.is_empty(), "unchanged markers must not be rewritten" ); }); } /// An appended entry and its value read back from a fresh journal. #[test_traced] fn test_oversized_append_and_get() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); // Append entry with value let value: TestValue = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let entry = TestEntry::new(42, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); assert_eq!(position, 0); // Get entry let retrieved_entry = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(retrieved_entry.id, 42); // Get value let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery truncates the index at the first entry whose value bytes were lost and /// keeps the entries before it. #[test_traced] fn test_oversized_tracked_recovery_truncates_lost_value_tail() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate oversized journal let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append multiple entries let mut locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Simulate crash: truncate glob to lose last 2 values let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Calculate size to keep first 3 entries let keep_size = byte_end(locations[2].1, locations[2].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery truncates the invalid index suffix. let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 3 entries should still be valid for i in 0..3u8 { let (position, offset, size) = locations[i as usize]; let entry = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); let value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(value, [i; 16]); } // Entry at position 3 should fail (index was truncated) let result = oversized.get(1, 3).await; assert!(result.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// A synced entry and its value survive a restore at the synced boundary. #[test_traced] fn test_oversized_persistence() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(123, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Restore at the synced boundary and verify. let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), cfg, (1, ENTRY_SIZE)) .await .expect("reinitialization must succeed"); let retrieved_entry = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(retrieved_entry.id, 123); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Syncing selected sections makes both journals of exactly those sections durable and skips a /// section that does not exist. #[test_traced] fn test_oversized_sync() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .expect("Failed to init"); // One sub-page entry/value per section stays buffered until synced. let mut located = Vec::new(); for section in 1u64..=3 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(section, entry, &value) .await .expect("Failed to append"); located.push((section, position, offset, size, value)); } // Sync sections 1 and 3 (both index and values); a nonexistent section (99) is // skipped, not an error. oversized = oversized .sync(&[1, 3, 99]) .await .expect("Failed to sync sections"); drop(oversized); // Only the synced sections survive the unclean drop, with both index and value durable. let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), cfg, (3, ENTRY_SIZE)) .await .expect("reinitialization must succeed"); for &(section, position, offset, size, value) in &located { let result = oversized.get(section, position).await; if section == 2 { assert!(result.is_err(), "unsynced section 2 must not be durable"); continue; } assert_eq!(result.expect("synced entry durable").id, section); let retrieved = oversized .get_value(section, offset, size) .await .expect("synced value durable"); assert_eq!(retrieved, value); } oversized.destroy().await.expect("Failed to destroy"); }); } /// Assert that every entry recovery adopted in section 1 reads back the value that was /// appended with it. async fn assert_adopted_entries_consistent( oversized: &Oversized, ) { let chunk = FixedJournal::::CHUNK_SIZE as u64; for position in 0..oversized.size(1).expect("size") / chunk { let entry = oversized.get(1, position).await.expect("Failed to get"); let (offset, size) = entry.value_location(); let value = oversized .get_value(1, offset, size) .await .expect("adopted entry must reference durable bytes"); assert_eq!( value, [entry.id as u8; 16], "entry {} must read back the value appended with it", entry.id ); } } /// A tracked open adopts a marker-covered entry without reading its damaged value. Only that /// value's own read fails, and the index bytes stay untouched. #[test] fn test_tracked_open_adopts_corrupt_committed_value_lazily() { deterministic::Runner::default().start(|context| async move { // Seed two durable entries and publish a marker covering both. let cfg = test_cfg(); let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("seed"), cfg.clone(), (0, 0)) .await .unwrap(); (journal, _, _, _) = journal .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .unwrap(); let offset; (journal, _, offset, _) = journal .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .unwrap(); _ = journal.sync_all().await.unwrap(); let mut metadata = Metadata::<_, SectionKey, u64>::init( context.child("markers"), MetadataConfig { partition: MARKERS.into(), codec_config: (), }, ) .await .unwrap(); metadata.put(SectionKey::new(1), 2); _ = metadata.sync().await.unwrap(); // Damage the second committed value in place and record the index bytes. let (value_blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .unwrap(); value_blob .write_at(offset, vec![0xff], WriteOptions::SYNC) .await .unwrap(); drop(value_blob); let (index_blob, size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .unwrap(); let before = index_blob .read_at(0, size as usize, ReadOptions::default()) .await .unwrap() .coalesce(); drop(index_blob); // The open adopts both marker-covered entries, and only the damaged value fails. let journal = open_tracked(context.child("reopen"), cfg.clone()) .await .unwrap(); assert_eq!(journal.size(1).unwrap(), ENTRY_SIZE * 2); let entry = journal.get(1, 1).await.unwrap(); let (value_offset, value_size) = entry.value_location(); assert!( journal .get_value(1, value_offset, value_size) .await .is_err(), "a damaged committed value must fail its read" ); drop(journal); // The index bytes are unchanged. let (index_blob, retained_size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .unwrap(); assert_eq!(retained_size, size); let after = index_blob .read_at(0, size as usize, ReadOptions::default()) .await .unwrap() .coalesce(); assert_eq!(before.as_ref(), after.as_ref()); }); } /// Tracked recovery durably drops an entry whose value was lost, so a later append that reuses /// its offset cannot revive it. #[test_traced] fn test_oversized_recovery_never_adopts_entries_for_lost_values() { // Crash 1: entry 1 becomes durable but its value does not (an index write surviving // a crash its value bytes did not). let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), test_cfg()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); oversized.index = oversized.index.sync(1).await.expect("Failed to sync index"); }); // Boot 2: recovery truncates entry 1 (its range is out of bounds) and must make that // truncation durable. A new append then reuses entry 1's offset. Crash 2 lands after the // value sync and before the index sync. let (_, checkpoint) = deterministic::Runner::from(checkpoint).start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); assert_eq!( oversized.size(1).expect("size"), 0, "entry without durable value bytes must be truncated" ); (oversized, _, _, _) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized.values = oversized .values .sync(1) .await .expect("Failed to sync values"); }); // Boot 3: without a durable truncation in recovery, the index would still hold // entry 1, now range-valid over entry 2's bytes. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), test_cfg()) .await .expect("reinitialization must succeed"); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery fails when its value truncation cannot be made durable, after its index /// truncation already is. #[test_traced] fn test_oversized_tracked_recovery_fails_when_truncation_cannot_be_made_durable() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // Two durable entry/value pairs with no marker covering them. let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); let (offset, size); (oversized, _, offset, size) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Damage entry 2's value in place, so recovery drops entry 2 and must release the // value bytes past entry 1. let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open values blob"); blob.write_at( offset, vec![0xFF; Widen::::widen(size)], WriteOptions::SYNC, ) .await .expect("Failed to corrupt value"); drop(blob); // Boot 2: the glob cannot be synced, so initialization must fail rather than return // with a values truncation that is not durable (later appends could otherwise reuse // entry 2's still-durable value range). let faulty_values = SyncFaultContext { inner: context.child("second"), fail_partition: "test-values".into(), }; assert!( open_tracked(faulty_values, cfg).await.is_err(), "recovery must fail if its value truncation is not durable" ); }); // The index truncation was made durable before the failure, so the dropped entry // must not be adopted at recovery. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), test_cfg()) .await .expect("reinitialization must succeed"); assert_eq!(oversized.size(1).expect("size"), ENTRY_SIZE); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery makes its value truncation durable before a later append can reuse the /// freed offset. #[test_traced] fn test_oversized_recovery_glob_truncation_durable_before_offset_reuse() { // Crash 1: the index truncation is durable but the glob still holds entry 2's frame (the // state a crash between recovery's index and value truncations leaves behind). let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), test_cfg()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); let chunk = FixedJournal::::CHUNK_SIZE as u64; oversized.index = oversized .index .test_reopen_at_most(1, chunk) .await .expect("Failed to truncate index"); oversized.index = oversized.index.sync(1).await.expect("Failed to sync index"); }); // Boot 2: recovery truncates the glob to entry 1's end and must make that // truncation durable. Entry 3 (same size) then reuses entry 2's freed range. // Crash 2 lands after the index sync and before the values sync. let (_, checkpoint) = deterministic::Runner::from(checkpoint).start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(3, 0, 0), &[3; 16]) .await .expect("Failed to append"); oversized.index = oversized.index.sync(1).await.expect("Failed to sync index"); }); // Boot 3: without a durable glob truncation in recovery, entry 3 would be // adopted referencing entry 2's still-durable frame. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), test_cfg()) .await .expect("reinitialization must succeed"); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery of an index shortened without its values releases the orphaned value bytes. #[test_traced] fn test_oversized_truncate_crash_between_truncations_recovers_post_truncate() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // Two fully durable entry/value pairs. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), test_cfg()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // Stop a recovery truncation after shortening the index but before the values. The // index truncation is durable but the freed value bytes are not yet truncated. let chunk = FixedJournal::::CHUNK_SIZE as u64; oversized.index = oversized .index .test_reopen_at_most(1, chunk) .await .expect("Failed to truncate index"); oversized.index = oversized.index.sync(1).await.expect("Failed to sync index"); }); // Recovery must truncate the orphaned value bytes and land on the post-truncate // state. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); let chunk = FixedJournal::::CHUNK_SIZE as u64; assert_eq!(oversized.size(1).expect("size"), chunk); let entry = oversized.get(1, 0).await.expect("Failed to get"); let (offset, size) = entry.value_location(); assert_eq!( oversized.values.size(1).expect("glob size"), byte_end(offset, size), "orphaned value bytes must be truncated" ); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// A completed start_sync handle makes both journals of its section durable. #[test_traced] fn test_oversized_start_sync_completion_means_recoverable() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); let (_oversized, handle) = oversized.start_sync(1).await.expect("Failed to start sync"); handle.await.expect("sync must complete"); // Crash: everything covered by the completed handle must survive. }); // Restore at the boundary the completed sync covered. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint( context.child("second"), test_cfg(), (1, ENTRY_SIZE), ) .await .expect("reinitialization must succeed"); let chunk = FixedJournal::::CHUNK_SIZE as u64; assert_eq!(oversized.size(1).expect("size"), chunk); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// A failed value sync surfaces as an error, and tracked recovery of that storage drops the /// entry whose value never became durable. #[test_traced] fn test_oversized_sync_values_failure_recovers_clean() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // Open a fresh journal whose value syncs fail. let faulty_values = SyncFaultContext { inner: context.child("first"), fail_partition: "test-values".into(), }; let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(faulty_values, test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); // The value sync fails, so the caller is never acknowledged. The index sync // may still land, but recovery must not adopt an entry whose value bytes never // became durable. assert!(oversized.sync(1).await.is_err(), "value sync must fail"); }); // No acknowledged sync covers the entry. A tracked open of storage written without // markers validates every value. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); assert_eq!( oversized.size(1).expect("size"), 0, "entry without durable value bytes must be truncated" ); oversized.destroy().await.expect("Failed to destroy"); }); } /// A failed value sync surfaces through the start_sync handle, and tracked recovery of that /// storage drops the entry whose value never became durable. #[test_traced] fn test_oversized_start_sync_values_failure_recovers_clean() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // Open a fresh journal whose value syncs fail. let faulty_values = SyncFaultContext { inner: context.child("first"), fail_partition: "test-values".into(), }; let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(faulty_values, test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); // The value sync fails, so the handle must surface the failure and the caller // is never acknowledged. let (_oversized, handle) = oversized.start_sync(1).await.expect("Failed to start sync"); assert!(handle.await.is_err(), "value sync must fail"); }); // No acknowledged sync covers the entry. A tracked open of storage written without // markers validates every value. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); assert_eq!( oversized.size(1).expect("size"), 0, "entry without durable value bytes must be truncated" ); oversized.destroy().await.expect("Failed to destroy"); }); } /// Dropping a start_sync handle does not cancel its sync, and the next sync completes the work. #[test_traced] fn test_oversized_start_sync_dropped_handle_driven_by_next_sync() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); // Drop the handle without observing it: the next sync must wait for the // started syncs and complete the work. let (oversized, handle) = oversized.start_sync(1).await.expect("Failed to start sync"); drop(handle); oversized.sync(1).await.expect("Failed to sync"); }); // Restore at the boundary the completed sync covered. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint( context.child("second"), test_cfg(), (1, ENTRY_SIZE), ) .await .expect("reinitialization must succeed"); let chunk = FixedJournal::::CHUNK_SIZE as u64; assert_eq!(oversized.size(1).expect("size"), chunk); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery rejects an entry whose value range is in bounds but fails its checksum. #[test_traced] fn test_oversized_recovery_rejects_entry_with_torn_value_bytes() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // One fully durable entry/value pair. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), test_cfg()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // Append entry 2, make its index entry durable, then persist the glob's LENGTH // over entry 2's range without its bytes (writeback-mode metadata journaling): // overwrite the frame with same-length garbage and sync the values journal. let (offset, size); (oversized, _, offset, size) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized.index = oversized.index.sync(1).await.expect("Failed to sync index"); oversized .values .inject(1, offset, vec![0xFF; size as usize]) .await .expect("Failed to overwrite value bytes"); oversized.values = oversized .values .sync(1) .await .expect("Failed to sync values"); }); // Entry 2's range fits within the glob, so only the checksum check can reject it. deterministic::Runner::from(checkpoint).start(|context| async move { let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), test_cfg()) .await .expect("reinitialization must succeed"); let chunk = FixedJournal::::CHUNK_SIZE as u64; assert_eq!( oversized.size(1).expect("size"), chunk, "entry with torn value bytes must be truncated" ); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery truncates a section at a torn interior index page, even though a valid /// final page hides it from tail discovery and every value after it is intact. #[test_traced] fn test_recovery_truncates_at_torn_interior_index_page() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Use page size = entry size so each entry is on its own page. let cfg = entry_cfg(); // Create five durable entry/value pairs. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for i in 1..=5u8 { (oversized, _, _, _) = oversized .append(1, TestEntry::new(i as u64, 0, 0), &[i; 16]) .await .expect("Failed to append"); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Corrupt the CRC record of the third entry's index page. Tail discovery stops at the // valid last page, so the torn page survives in bounds. let physical_page = ENTRY_SIZE + 12; let (index_blob, size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open index blob"); assert_eq!(size, 5 * physical_page); index_blob .write_at( 2 * physical_page + ENTRY_SIZE, vec![0xFF; 12], WriteOptions::SYNC, ) .await .expect("Failed to corrupt index page"); drop(index_blob); // Recovery keeps only the entries before the torn page and releases the values of the // entries it drops. let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); let chunk = FixedJournal::::CHUNK_SIZE as u64; assert_eq!(oversized.size(1).expect("size"), 2 * chunk); assert_adopted_entries_consistent(&oversized).await; let last = oversized.get(1, 1).await.expect("Failed to get"); assert_eq!( oversized.values.size(1).expect("glob size"), byte_end(last.value_offset, last.value_size), "values of dropped entries must be released" ); oversized.destroy().await.expect("Failed to destroy"); }); } /// Restore truncates to the checkpoint and removes later sections without validating the state /// it discards. #[test_traced] fn test_oversized_restore_discards_beyond_checkpoint() { let executor = deterministic::Runner::default(); let (_, checkpoint) = executor.start_and_recover(|context| async move { // One committed entry, then torn state beyond the checkpoint: entries in // section 1 and 2 whose index becomes durable ahead of their values. let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(2, TestEntry::new(3, 0, 0), &[3; 16]) .await .expect("Failed to append"); oversized.index = oversized .index .sync(&[1, 2]) .await .expect("Failed to sync index"); }); // Restore truncates to the checkpoint without validating the discarded state. deterministic::Runner::from(checkpoint).start(|context| async move { let chunk = FixedJournal::::CHUNK_SIZE as u64; let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), test_cfg(), (1, chunk)) .await .expect("reinitialization must succeed"); assert_eq!(oversized.size(1).expect("size"), chunk); assert_eq!(oversized.newest_section(), Some(1)); assert_adopted_entries_consistent(&oversized).await; oversized.destroy().await.expect("Failed to destroy"); }); } /// Restore removes the sections past its checkpoint without opening or repairing them. #[test_traced] fn test_oversized_restore_does_not_repair_discarded_sections() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = entry_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("seed"), cfg.clone(), (0, 0)) .await .expect("failed to init"); // Section 1 holds the only checkpointed entry. Sections 2 and 3 lie past the // checkpoint. Each index page holds one entry, so section 2's three entries give // corrupt_page an interior page 1 that a complete page follows. (oversized, _, _, _) = oversized .append(1, TestEntry::new(0, 0, 0), &[0; 16]) .await .expect("failed to append checkpoint entry"); for id in 1..=3 { (oversized, _, _, _) = oversized .append(2, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .expect("failed to append discardable entry"); } (oversized, _, _, _) = oversized .append(3, TestEntry::new(4, 0, 0), &[4; 16]) .await .expect("failed to append discardable entry"); oversized = oversized.sync_all().await.expect("failed to sync"); drop(oversized); // Restore owns no bytes in sections 2 and 3. The valid final page hides section 2's // interior hole from paged tail recovery, while section 3's torn tail would be // repaired and synced if recovery opened it. corrupt_page( &context, &cfg.index_partition, &2u64.to_be_bytes(), 1, ENTRY_SIZE, ) .await; tear_index_tail(&context, &cfg, 3).await; // Live exclusive handles detect a discarded section that restore mistakenly opens. // The value buffer does no I/O on open, so only a handle observes that section. let (discarded_index, _) = context .open(&cfg.index_partition, &2u64.to_be_bytes()) .await .unwrap(); let (discarded_value, _) = context .open(&cfg.value_partition, &2u64.to_be_bytes()) .await .unwrap(); let pending = PendingSyncs::default(); pending.arm(); let delayed = DelayedSyncContext { inner: context, pending: pending.clone(), }; let chunk = ENTRY_SIZE; let oversized: Oversized<_, TestEntry, TestValue> = drive_pending_syncs( &pending, Oversized::init_with_checkpoint(delayed.child("restore"), cfg.clone(), (1, chunk)), ) .await .expect("checkpoint restore failed"); // Restore syncs both retained sections. Repairing the torn discarded index // section would add another sync. assert_eq!(pending.calls(), 2); drop(discarded_index); drop(discarded_value); // Restore removed sections 2 and 3 from both partitions without opening them. let retained = vec![1u64.to_be_bytes().to_vec()]; assert_eq!(delayed.scan(&cfg.index_partition).await.unwrap(), retained); assert_eq!(delayed.scan(&cfg.value_partition).await.unwrap(), retained); oversized.destroy().await.expect("failed to destroy"); }); } /// A tracked open trims a torn index tail above the marker and durably releases the value the /// trimmed entry referenced. #[test_traced] fn test_oversized_tracked_open_repairs_torn_index_tail() { deterministic::Runner::default().start(|context| async move { // Seed two durable one-page entries and publish a marker covering only the first. let cfg = entry_cfg(); let mut journal = open_tracked(context.child("seed"), cfg.clone()) .await .unwrap(); for id in 0..2 { (journal, _, _, _) = journal .append(1, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } journal = journal.sync_all().await.unwrap(); let mut tracking = journal.tracking.take().unwrap(); assert!(tracking.stage_marker(1, 1)); tracking.metadata = tracking.metadata.sync().await.unwrap(); drop(tracking); drop(journal); // Tearing the tail truncates entry 1's physical page above the marker, so paged // recovery trims it when opening section 1. tear_index_tail(&context, &cfg, 1).await; // Reopen while counting the syncs the open performs. let pending = PendingSyncs::default(); pending.arm(); let delayed = DelayedSyncContext { inner: context, pending: pending.clone(), }; let chunk = ENTRY_SIZE; let journal = drive_pending_syncs(&pending, open_tracked(delayed.child("reopen"), cfg)) .await .unwrap(); // One sync makes the index trim durable. The other makes truncating entry 1's value // durable, since no retained index entry references that value. The marker already // matches the retained entry, so no marker generation is written. assert_eq!(pending.calls(), 2); assert_eq!(journal.size(1).unwrap(), chunk); assert_eq!(journal.last(1).await.unwrap().unwrap().id, 0); journal.destroy().await.unwrap(); }); } /// Tracked recovery of an image whose newest index sections are gone while every value section /// remains keeps each surviving entry with its value, removes the orphaned value sections, and /// restarts the removed sections from zero. The cases lose only the newest index section and /// every index section above section 1. #[rstest::rstest] #[case::newest(3)] #[case::all(1)] #[test_traced] fn test_oversized_tracked_open_recovers_interrupted_section_removal(#[case] kept: u64) { const SECTIONS: u64 = 4; const ENTRIES: u64 = 3; deterministic::Runner::default().start(|context| async move { // Seed sections 1 through 4 with three entries each, every entry with a distinct // value, and make both journals durable. let cfg = entry_cfg(); let mut journal: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("seed"), cfg.clone()) .await .unwrap(); for section in 1..=SECTIONS { for position in 0..ENTRIES { let id = section * ENTRIES + position; (journal, _, _, _) = journal .append(section, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } } journal = journal.sync_all().await.unwrap(); drop(journal); // Remove every index section above `kept` and leave every value section. for section in (kept + 1..=SECTIONS).rev() { context .remove(&cfg.index_partition, Some(§ion.to_be_bytes())) .await .unwrap(); } // A tracked open retains every surviving index entry with its value. let mut journal: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("restart"), cfg.clone()) .await .unwrap(); assert_eq!(journal.newest_section(), Some(kept)); for section in 1..=kept { assert_eq!(journal.size(section).unwrap(), ENTRIES * ENTRY_SIZE); for position in 0..ENTRIES { let id = section * ENTRIES + position; let entry = journal.get(section, position).await.unwrap(); assert_eq!(entry.id, id); assert_eq!( journal .get_value(section, entry.value_offset, entry.value_size) .await .unwrap(), [id as u8; 16] ); } } // Removed sections stay out of range, and recovery removes the value sections that // lost their index, so both partitions hold exactly the surviving sections. for section in kept + 1..=SECTIONS { assert!(matches!( journal.get(section, 0).await, Err(Error::SectionOutOfRange(missing)) if missing == section )); } let retained: Vec<_> = (1..=kept) .map(|section| section.to_be_bytes().to_vec()) .collect(); assert_eq!(context.scan(&cfg.index_partition).await.unwrap(), retained); assert_eq!(context.scan(&cfg.value_partition).await.unwrap(), retained); // Appending to the newest surviving section continues after its last entry and value. let end = journal.value_size(kept).await.unwrap(); let (next, offset); (journal, next, offset, _) = journal .append(kept, TestEntry::new(100, 0, 0), &[100; 16]) .await .unwrap(); assert_eq!(next, ENTRIES); assert_eq!(offset, end); // Appending to the first removed section starts both of its journals from zero. let (first, start); (journal, first, start, _) = journal .append(kept + 1, TestEntry::new(101, 0, 0), &[101; 16]) .await .unwrap(); assert_eq!(first, 0); assert_eq!(start, 0); journal = journal.sync_all().await.unwrap(); drop(journal); // A clean reopen keeps the recovered prefix and both appended entries. let journal: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("reopen"), cfg).await.unwrap(); assert_eq!(journal.newest_section(), Some(kept + 1)); for section in 1..kept { assert_eq!(journal.size(section).unwrap(), ENTRIES * ENTRY_SIZE); } assert_eq!(journal.size(kept).unwrap(), (ENTRIES + 1) * ENTRY_SIZE); for (section, position, id) in [(kept, ENTRIES, 100), (kept + 1, 0, 101)] { let entry = journal.get(section, position).await.unwrap(); assert_eq!(entry.id, id); assert_eq!( journal .get_value(section, entry.value_offset, entry.value_size) .await .unwrap(), [id as u8; 16] ); } journal.destroy().await.unwrap(); }); } /// Restore fails without repairing anything when a section the checkpoint covers lost value /// bytes. #[test_traced] fn test_oversized_restore_incomplete_section_errors() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Two committed sections let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(2, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Truncate section 1's values, simulating lost durable state below the // checkpoint let (blob, len) = context .open("test-values", &1u64.to_be_bytes()) .await .expect("Failed to open values blob"); blob.resize(len - 1).await.expect("Failed to resize"); blob.sync().await.expect("Failed to sync"); drop(blob); // The checkpoint covers the damaged section, so init must fail. Nothing is // repaired, so the failure persists across restarts. let chunk = FixedJournal::::CHUNK_SIZE as u64; for instance in ["second", "third"] { let result: Result, Error> = Oversized::init_with_checkpoint( context.child(instance), test_cfg(), (2, chunk), ) .await; assert!(matches!(result, Err(Error::Corruption(_)))); } }); } /// Restore adopts interior index damage below a checkpoint boundary. The damage surfaces as a /// read error and stays byte-for-byte in place. #[test_traced] fn test_oversized_restore_adopts_interior_index_corruption() { let executor = deterministic::Runner::default(); executor.start(|context| async move { for (child, seed_child, restore_child, checkpoint) in [ ( "checkpoint_section", "seed_checkpoint_section", "restore_checkpoint_section", (1, 3 * ENTRY_SIZE), ), ( "earlier_section", "seed_earlier_section", "restore_earlier_section", (2, ENTRY_SIZE), ), ] { let mut cfg = entry_cfg(); cfg.index_partition = format!("test-index-{child}"); cfg.value_partition = format!("test-values-{child}"); // Persist three entries in section 1 and a later checkpoint candidate. One entry // per integrity page makes the damaged page strictly interior. let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child(seed_child), cfg.clone(), (0, 0)) .await .expect("failed to init"); for id in 0..3 { (oversized, _, _, _) = oversized .append(1, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .expect("failed to append"); } (oversized, _, _, _) = oversized .append(2, TestEntry::new(3, 0, 0), &[3; 16]) .await .expect("failed to append later section"); oversized = oversized.sync_all().await.expect("failed to sync"); drop(oversized); // Checkpoint recovery trusts the completed durability boundary and reads only its // terminal entry. Arbitrary post-commit bit rot remains a lazy read error. corrupt_page( &context, &cfg.index_partition, &1u64.to_be_bytes(), 1, ENTRY_SIZE, ) .await; let (blob, expected_size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("failed to open index"); let expected = blob .read_at(0, expected_size as usize, ReadOptions::default()) .await .expect("failed to snapshot damaged index") .coalesce(); drop(blob); let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint( context.child(restore_child), cfg.clone(), checkpoint, ) .await .expect("checkpoint restore should adopt the covered prefix"); assert!(matches!( oversized.get(1, 1).await, Err(Error::Runtime(RError::InvalidChecksum)) )); drop(oversized); let (blob, actual_size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("failed to reopen index"); assert_eq!(actual_size, expected_size); let actual = blob .read_at(0, actual_size as usize, ReadOptions::default()) .await .expect("failed to read damaged index") .coalesce(); assert_eq!(actual.as_ref(), expected.as_ref()); } }); } /// Restore adopts a section without reading its values, so a rotted committed value fails only /// its own read. #[test_traced] fn test_oversized_restore_adopts_rotted_committed_value() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Two committed sections let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .expect("Failed to init"); let (offset, size); (oversized, _, offset, size) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(2, TestEntry::new(2, 0, 0), &[2; 16]) .await .expect("Failed to append"); oversized = oversized.sync_all().await.expect("Failed to sync"); // Corrupt entry 1's committed value in place (sizes unchanged) oversized .values .inject(1, offset, vec![0xFF; size as usize]) .await .expect("Failed to corrupt value"); oversized.values = oversized.values.sync(1).await.expect("Failed to sync"); drop(oversized); // Restore adopts the section without probing its values: the corruption // surfaces at read on exactly the affected entry. let chunk = FixedJournal::::CHUNK_SIZE as u64; let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), test_cfg(), (2, chunk)) .await .expect("reinitialization must succeed"); assert!(matches!( oversized.get_value(1, offset, size).await, Err(Error::ChecksumMismatch(_, _)) )); let entry = oversized.get(2, 0).await.expect("Failed to get"); let (offset, size) = entry.value_location(); assert_eq!( oversized .get_value(2, offset, size) .await .expect("Failed to get value"), [2; 16] ); oversized.destroy().await.expect("Failed to destroy"); }); } /// Finishing a tracked startup replay before it is drained fails. #[test_traced] fn test_tracked_replay_finish_before_drain_fails() { deterministic::Runner::default().start(|context| async move { // Seed one durable entry. let cfg = test_cfg(); let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("seed"), cfg.clone(), (0, 0)) .await .unwrap(); (journal, _, _, _) = journal .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .unwrap(); _ = journal.sync_all().await.unwrap(); // A replay that has not yielded its entry must not finish. let replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("oversized"), cfg.clone(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); assert!(matches!(replay.finish().await, Err(Error::ReplayFailed))); // The synced entry reopens through its checkpoint, so the store can be destroyed. Oversized::<_, TestEntry, TestValue>::init_with_checkpoint( context, cfg, (1, ENTRY_SIZE), ) .await .unwrap() .destroy() .await .unwrap(); }); } /// The tracked startup replay passes the caller's read options to every index read it makes /// once the index writers are open. #[test_traced] fn test_oversized_replay_propagates_read_options() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Seed one durable entry and publish a marker covering it, so the replay reads no // value. let cfg = entry_cfg(); let page_cache = cfg.index_page_cache.clone(); let mut oversized = open_tracked(context.child("seed"), cfg.clone()) .await .expect("Failed to init"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(1, 0, 0), &[1; 16]) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); let mut tracking = oversized.tracking.take().unwrap(); assert!(tracking.stage_marker(1, 1)); tracking.metadata = tracking.metadata.sync().await.unwrap(); drop(tracking); drop(oversized); // Reopen and evict the index page so replay must exercise the backing journal read. let (context, recordings) = commonware_runtime::mocks::RecordingContext::new(context); let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("reopen"), cfg, MARKERS.into(), ReadOptions::DONT_CACHE, ) .await .expect("Failed to replay"); page_cache.clear(); recordings.clear(); // The first call opens the index writer, which reads its tail page once. The lazy // refill follows and must preserve the caller's policy. let (section, position, entry) = replay .next() .await .expect("missing replay item") .expect("Failed to read replay item"); assert_eq!((section, position, entry.id), (1, 0, 1)); let reads = recordings.snapshot().reads; let (opened, replayed) = reads.split_at(1); assert_eq!(opened, [ReadOptions::default()]); assert!( !replayed.is_empty(), "the replay must read the evicted page" ); assert!( replayed .iter() .all(|options| *options == ReadOptions::DONT_CACHE), "every replay read must use the caller's options" ); assert!(replay.next().await.is_none()); replay .finish() .await .expect("failed to finish replay") .destroy() .await .expect("Failed to destroy"); }); } /// Pruning removes every section below the floor and keeps the rest. #[test_traced] fn test_oversized_prune() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); // Append to multiple sections for section in 1u64..=5 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(section).await.expect("Failed to sync"); } // Prune sections < 3 (oversized, _) = oversized.prune(3).await.expect("Failed to prune"); // The public accessor mirrors the guard assert!(oversized.pruned(1)); assert!(oversized.pruned(2)); assert!(!oversized.pruned(3)); // Sections 1, 2 should be gone assert!(oversized.get(1, 0).await.is_err()); assert!(oversized.get(2, 0).await.is_err()); // Sections 3, 4, 5 should exist assert!(oversized.get(3, 0).await.is_ok()); assert!(oversized.get(4, 0).await.is_ok()); assert!(oversized.get(5, 0).await.is_ok()); oversized.destroy().await.expect("Failed to destroy"); }); } /// A restore whose checkpoint names the only written section retains its entry. #[test_traced] fn test_checkpoint_restore_of_sole_section() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create oversized journal let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .expect("Failed to init"); // Write and sync section 2 only. let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(2, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(2).await.expect("Failed to sync"); drop(oversized); // A restore at section 2's synced boundary needs no earlier section. let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), cfg, (2, ENTRY_SIZE)) .await .expect("reinitialization must succeed"); // Section 2 entry should be valid let entry = oversized.get(2, 0).await.expect("Failed to get"); assert_eq!(entry.id, 1); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery empties a section whose values were all lost, and appends restart at /// position zero. #[test_traced] fn test_recovery_all_entries_invalid() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append 5 entries for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Truncating glob to 0 bytes invalidates every entry let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(0).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery truncates the invalid index to zero entries. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // No entries should be accessible let result = oversized.get(1, 0).await; assert!(result.is_err()); // Should be able to append after recovery let value: TestValue = [99; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 0); let retrieved = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery truncates each section independently at its first lost value. #[test_traced] fn test_recovery_multiple_sections_mixed_validity() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate multiple sections let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Section 1: 3 entries let mut section1_locations = Vec::new(); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); section1_locations.push((position, offset, size)); } // Section 2: 5 entries let mut section2_locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [10 + i; 16]; let entry = TestEntry::new(10 + i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(2, entry, &value) .await .expect("Failed to append"); section2_locations.push((position, offset, size)); } // Section 3: 2 entries for i in 0..2u8 { let value: TestValue = [20 + i; 16]; let entry = TestEntry::new(20 + i as u64, 0, 0); (oversized, _, _, _) = oversized .append(3, entry, &value) .await .expect("Failed to append"); } // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Truncate section 1 glob to keep only first entry let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let keep_size = byte_end(section1_locations[0].1, section1_locations[0].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Truncate section 2 glob to keep first 3 entries let (blob, _) = context .open(&cfg.value_partition, &2u64.to_be_bytes()) .await .expect("Failed to open blob"); let keep_size = byte_end(section2_locations[2].1, section2_locations[2].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Section 3 remains intact // Reinitialize let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // Section 1: only position 0 valid assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(1, 1).await.is_err()); assert!(oversized.get(1, 2).await.is_err()); // Section 2: positions 0,1,2 valid assert!(oversized.get(2, 0).await.is_ok()); assert!(oversized.get(2, 1).await.is_ok()); assert!(oversized.get(2, 2).await.is_ok()); assert!(oversized.get(2, 3).await.is_err()); assert!(oversized.get(2, 4).await.is_err()); // Section 3: both positions valid assert!(oversized.get(3, 0).await.is_ok()); assert!(oversized.get(3, 1).await.is_ok()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops an entry whose index page fails its checksum, and appends follow the /// remaining entries. #[test_traced] fn test_recovery_corrupted_last_index_entry() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Use page size = entry size so each entry is on its own page. // This allows corrupting just the last entry's page without affecting others. // Physical page size = TestEntry::SIZE (20) + 12 (CRC record) = 32 bytes. let cfg = entry_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append 5 entries (each on its own page) for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Corrupt the last page's CRC to trigger page-level integrity failure let (blob, size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Physical page size = 20 + 12 = 32 bytes // 5 entries = 5 pages = 160 bytes total // Last page CRC starts at offset 160 - 12 = 148 assert_eq!(size, 160); let last_page_crc_offset = size - 12; blob.write_at(last_page_crc_offset, vec![0xFF; 12], WriteOptions::SYNC) .await .expect("Failed to corrupt"); drop(blob); // Reinitializing should detect page corruption and truncate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // First 4 entries should be valid (on pages 0-3) for i in 0..4u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // Entry 4 should be gone (its page was corrupted) assert!(oversized.get(1, 4).await.is_err()); // Should be able to append after recovery let value: TestValue = [99; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 4); let retrieved = oversized.get(1, 4).await.expect("Failed to get"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// A restore at the newest synced boundary keeps every entry of every section. #[test_traced] fn test_checkpoint_restore_keeps_every_synced_entry() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .expect("Failed to init"); // Append entries to multiple sections for section in 1u64..=3 { for i in 0..10u8 { let value: TestValue = [(section as u8) * 10 + i; 16]; let entry = TestEntry::new(section * 100 + i as u64, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); } oversized = oversized.sync(section).await.expect("Failed to sync"); } drop(oversized); // Restore at the newest synced boundary. let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), cfg, (3, 10 * ENTRY_SIZE)) .await .expect("reinitialization must succeed"); // All entries should be valid for section in 1u64..=3 { for i in 0..10u8 { let entry = oversized .get(section, i as u64) .await .expect("Failed to get"); assert_eq!(entry.id, section * 100 + i as u64); } } oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops the only entry of a section whose value was lost. #[test_traced] fn test_recovery_single_entry_invalid() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with single entry let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Truncating glob to 0 invalidates the single entry let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(0).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitialize let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // Entry should be gone assert!(oversized.get(1, 0).await.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops an entry whose value is one byte short. #[test_traced] fn test_recovery_last_entry_off_by_one() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Truncate glob to be off by 1 byte from last entry let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Last entry needs: offset + size bytes // Truncate to offset + size - 1 (missing 1 byte) let last = &locations[2]; let truncate_to = byte_end(last.1, last.2) - 1; blob.resize(truncate_to).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitialize let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // First 2 entries should be valid assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(1, 1).await.is_ok()); // Entry 2 should be gone (truncated) assert!(oversized.get(1, 2).await.is_err()); // Should be able to append after recovery let value: TestValue = [99; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 2); let retrieved = oversized.get(1, 2).await.expect("Failed to get"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops every entry of a section whose value blob is missing. #[test_traced] fn test_recovery_glob_missing_entirely() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Delete the glob file entirely context .remove(&cfg.value_partition, Some(&1u64.to_be_bytes())) .await .expect("Failed to remove"); // Reinitialize with an empty glob, so every entry is invalid let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // All entries should be gone assert!(oversized.get(1, 0).await.is_err()); assert!(oversized.get(1, 1).await.is_err()); assert!(oversized.get(1, 2).await.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Appends after tracked recovery continue at the truncated position. #[test_traced] fn test_recovery_can_append_after_recovery() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Truncate glob to keep only first 2 entries let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let keep_size = byte_end(locations[1].1, locations[1].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitialize let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Verify first 2 entries exist assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(1, 1).await.is_ok()); assert!(oversized.get(1, 2).await.is_err()); // Append new entries after recovery for i in 10..15u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); } oversized = oversized.sync(1).await.expect("Failed to sync"); // Verify new entries at positions 2, 3, 4, 5, 6 for i in 0..5u8 { let entry = oversized .get(1, 2 + i as u64) .await .expect("Failed to get new entry"); assert_eq!(entry.id, (10 + i) as u64); } oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops the entries of a section whose value section is gone while its index /// remains. #[test_traced] fn test_recovery_glob_pruned_but_index_not() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate multiple sections let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for section in 1u64..=3 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); } // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Remove section 1's values while its index remains. use crate::journal::segmented::glob::{Config as GlobConfig, Glob}; let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); (glob, _) = glob.prune(2).await.expect("Failed to prune glob"); glob = glob.sync_all().await.expect("Failed to sync glob"); drop(glob); // Recovery truncates index section 1 to zero entries. let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Losing the values removes the corresponding index entries. assert!(oversized.get(1, 0).await.is_err()); // Sections 2 and 3 should still be valid assert!(oversized.get(2, 0).await.is_ok()); assert!(oversized.get(3, 0).await.is_ok()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery removes the value section of a lost index section and keeps the other /// sections. #[test_traced] fn test_recovery_index_partition_deleted() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate multiple sections let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for section in 1u64..=3 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); } // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Delete index blob for section 2 (simulate corruption/loss) context .remove(&cfg.index_partition, Some(&2u64.to_be_bytes())) .await .expect("Failed to remove index"); // Reinitializing should handle the missing index section gracefully // Section 2 is gone from index, orphan data in glob is acceptable let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Section 1 and 3 should still be valid assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(3, 0).await.is_ok()); // Section 2 should be gone (index file deleted) assert!(oversized.get(2, 0).await.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery truncates index entries that outlived their unsynced values. #[test_traced] fn test_recovery_index_synced_but_glob_not() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append entries and sync let mut locations = Vec::new(); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); // Add more entries WITHOUT syncing (simulates unsynced writes) for i in 10..15u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } // Note: NOT calling sync() here drop(oversized); // Simulate crash where index was synced but glob wasn't: // Truncate glob back to the synced size (3 entries) let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let synced_size = byte_end(locations[2].1, locations[2].2); blob.resize(synced_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery truncates the index to the glob boundary. let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // First 3 entries should be valid for i in 0..3u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // Entries 3-7 should be gone (unsynced, index truncated) assert!(oversized.get(1, 3).await.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery releases value bytes that outlived their index entries, and appends reuse /// the freed range. #[test_traced] fn test_recovery_glob_synced_but_index_not() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Use page size = entry size so each entry is exactly one page. // This allows truncating by entry count to equal truncating by full pages, // maintaining page-level integrity. let cfg = entry_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append entries and sync let mut locations = Vec::new(); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Simulate crash: truncate INDEX but leave GLOB intact // This creates orphan data in glob (glob ahead of index) let (blob, _size) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Keep only first 2 index entries (2 full pages) // Physical page size = logical (20) + CRC record (12) = 32 bytes let physical_page_size = (TestEntry::SIZE + 12) as u64; blob.resize(2 * physical_page_size) .await .expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitialize while glob holds orphan data from entry 3 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 2 entries should be valid for i in 0..2u8 { let (position, offset, size) = locations[i as usize]; let entry = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); let value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(value, [i; 16]); } // Entry at position 2 should fail (index was truncated) assert!(oversized.get(1, 2).await.is_err()); // Appending after recovery continues after the two valid entries let mut new_locations = Vec::new(); for i in 10..13u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); // New entries start at position 2 (after the 2 valid entries) assert_eq!(position, (i - 10 + 2) as u64); new_locations.push((position, offset, size, i)); // Verify we can read the new entry let retrieved = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(retrieved.id, i as u64); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); } // Sync and restart again to verify persistence with orphan data oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Reinitialize after adding data on top of orphan glob data let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), cfg) .await .expect("Failed to reinit after append"); // Read all valid entries in the index // First 2 entries from original data for i in 0..2u8 { let (position, offset, size) = locations[i as usize]; let entry = oversized.get(1, position).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); let value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(value, [i; 16]); } // New entries added after recovery for (position, offset, size, expected_id) in &new_locations { let entry = oversized .get(1, *position) .await .expect("Failed to get new entry after restart"); assert_eq!(entry.id, *expected_id as u64); let value = oversized .get_value(1, *offset, *size) .await .expect("Failed to get new value after restart"); assert_eq!(value, [*expected_id; 16]); } // Verify total entry count: 2 original + 3 new = 5 assert!(oversized.get(1, 4).await.is_ok()); assert!(oversized.get(1, 5).await.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops a partial index entry, and appends follow the complete entries. #[test_traced] fn test_recovery_partial_index_entry() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append 3 entries for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Simulate crash during write: truncate index to partial entry // Each entry is TestEntry::SIZE (20) + 4 (CRC32) = 24 bytes // Truncate to 3 full entries + 10 bytes of partial entry let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let partial_size = 3 * 24 + 10; // 3 full entries + partial blob.resize(partial_size).await.expect("Failed to resize"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitializing should handle the partial entry gracefully let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 3 entries should still be valid for i in 0..3u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // Entry 3 should not exist (partial entry was removed) assert!(oversized.get(1, 3).await.is_err()); // Append new entry after recovery let value: TestValue = [42; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 3); // Verify we can read the new entry let retrieved = oversized.get(1, 3).await.expect("Failed to get new entry"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get new value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery empties a section that holds only a partial index entry. #[test_traced] fn test_recovery_only_partial_entry() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with single entry let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Truncate index to only partial data (less than one full entry) let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(10).await.expect("Failed to resize"); // Less than chunk size blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery truncates the partial index to zero entries. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // No entries should exist assert!(oversized.get(1, 0).await.is_err()); // Should be able to append after recovery let value: TestValue = [99; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 0); let retrieved = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery of an index truncated ahead of its values keeps the shorter index. #[test_traced] fn test_recovery_crash_during_truncate_index_ahead() { // Simulates crash where index was truncated but glob wasn't let executor = deterministic::Runner::default(); executor.start(|context| async move { // Use page size = entry size so each entry is exactly one page. // This allows truncating by entry count to equal truncating by full pages, // maintaining page-level integrity. let cfg = entry_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Keep two index entries and leave values intact to model a crash between truncations. let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Physical page size = logical (20) + CRC record (12) = 32 bytes let physical_page_size = (TestEntry::SIZE + 12) as u64; blob.resize(2 * physical_page_size) .await .expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery on reinitialization should succeed (glob has orphan data) let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 2 entries should be valid for i in 0..2u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // Entries 2-4 should be gone (index was truncated) assert!(oversized.get(1, 2).await.is_err()); // Should be able to append new entries let pos; (oversized, pos, _, _) = oversized .append(1, TestEntry::new(100, 0, 0), &[100u8; 16]) .await .expect("Failed to append"); assert_eq!(pos, 2); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery of values truncated ahead of the index truncates the index to match. #[test_traced] fn test_recovery_crash_during_truncate_glob_ahead() { // Simulates crash where glob was truncated but index wasn't let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Keep two values and leave the index intact to model interrupted recovery. let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let keep_size = byte_end(locations[1].1, locations[1].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Recovery on reinitialization should detect index entries pointing beyond glob let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 2 entries should be valid (index truncated to match glob) for i in 0..2u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // Entries 2-4 should be gone (index truncated during recovery) assert!(oversized.get(1, 2).await.is_err()); // Should be able to append after recovery let value: TestValue = [99; 16]; let entry = TestEntry::new(100, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 2); let retrieved = oversized.get(1, 2).await.expect("Failed to get"); assert_eq!(retrieved.id, 100); let retrieved_value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, value); oversized.destroy().await.expect("Failed to destroy"); }); } /// Reading a value with a size smaller than its frame fails. #[test_traced] fn test_oversized_get_value_invalid_size() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); let (offset, _size); (oversized, _, offset, _size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // Size 0 should fail assert!(oversized.get_value(1, offset, 0).await.is_err()); // Size < value size should fail with codec error, checksum mismatch, or // insufficient length (if size < 4 bytes for checksum) for size in 1..4u32 { let result = oversized.get_value(1, offset, size).await; assert!( matches!( result, Err(Error::Codec(_)) | Err(Error::ChecksumMismatch(_, _)) | Err(Error::Runtime(_)) ), "expected error, got: {:?}", result ); } oversized.destroy().await.expect("Failed to destroy"); }); } /// Reading a value one byte short fails to decode or verify. #[test_traced] fn test_oversized_get_value_wrong_size() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); let (offset, correct_size); (oversized, _, offset, correct_size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // A size that is too small fails to decode or mismatches the checksum // (checksum mismatch can occur because we read wrong bytes as the checksum) let result = oversized.get_value(1, offset, correct_size - 1).await; assert!( matches!( result, Err(Error::Codec(_)) | Err(Error::ChecksumMismatch(_, _)) ), "expected Codec or ChecksumMismatch error, got: {:?}", result ); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery removes a value section that has no index section. #[test_traced] fn test_recovery_values_has_orphan_section() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with sections 1 and 2 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for section in 1u64..=2 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(section).await.expect("Failed to sync"); } drop(oversized); // Manually create an orphan value section (section 3) without corresponding index let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); let orphan_value: TestValue = [99; 16]; (glob, _, _) = glob .append(3, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(3).await.expect("Failed to sync glob"); drop(glob); // Reinitializing should detect and remove the orphan section let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Sections 1 and 2 should still be valid assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(2, 0).await.is_ok()); // Newest section should be 2 (orphan was removed) assert_eq!(oversized.newest_section(), Some(2)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery removes every value section that has no index section. #[test_traced] fn test_recovery_values_has_multiple_orphan_sections() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with only section 1 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [1; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Manually create multiple orphan value sections (2, 3, 4) let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); for section in 2u64..=4 { let orphan_value: TestValue = [section as u8; 16]; (glob, _, _) = glob .append(section, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(section).await.expect("Failed to sync glob"); } drop(glob); // Reinitializing should detect and remove all orphan sections let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Section 1 should still be valid assert!(oversized.get(1, 0).await.is_ok()); // Newest section should be 1 (orphans removed) assert_eq!(oversized.newest_section(), Some(1)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery of storage that holds only value sections removes them all. #[test_traced] fn test_recovery_index_empty_but_values_exist() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Manually create value sections without any index entries let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); for section in 1u64..=3 { let orphan_value: TestValue = [section as u8; 16]; (glob, _, _) = glob .append(section, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(section).await.expect("Failed to sync glob"); } drop(glob); // Initializing oversized should remove all orphan value sections let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // No sections should exist assert_eq!(oversized.newest_section(), None); assert_eq!(oversized.oldest_section(), None); oversized.destroy().await.expect("Failed to destroy"); }); } /// A section whose orphan value section tracked recovery removed accepts appends from zero. #[test_traced] fn test_recovery_orphan_section_append_after() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with section 1 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [1; 16]; let entry = TestEntry::new(1, 0, 0); let (offset1, size1); (oversized, _, offset1, size1) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Manually create orphan value sections (2, 3) let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); for section in 2u64..=3 { let orphan_value: TestValue = [section as u8; 16]; (glob, _, _) = glob .append(section, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(section).await.expect("Failed to sync glob"); } drop(glob); // Reinitializing should remove orphan sections let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Section 1 should still be valid let entry = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(entry.id, 1); let value = oversized .get_value(1, offset1, size1) .await .expect("Failed to get value"); assert_eq!(value, [1; 16]); // Should be able to append to section 2 after recovery let new_value: TestValue = [42; 16]; let new_entry = TestEntry::new(42, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(2, new_entry, &new_value) .await .expect("Failed to append after recovery"); assert_eq!(pos, 0); // Verify the new entry let retrieved = oversized.get(2, 0).await.expect("Failed to get"); assert_eq!(retrieved.id, 42); let retrieved_value = oversized .get_value(2, offset, size) .await .expect("Failed to get value"); assert_eq!(retrieved_value, new_value); // Sync and restart to verify persistence oversized = oversized.sync(2).await.expect("Failed to sync"); drop(oversized); let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), cfg) .await .expect("Failed to reinit after append"); // Both sections should be valid assert!(oversized.get(1, 0).await.is_ok()); assert!(oversized.get(2, 0).await.is_ok()); assert_eq!(oversized.newest_section(), Some(2)); oversized.destroy().await.expect("Failed to destroy"); }); } /// A restore at the newest synced boundary keeps every section when no value section is /// orphaned. #[test_traced] fn test_checkpoint_restore_keeps_every_synced_section() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate with sections 1, 2, 3 (no orphans) let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .expect("Failed to init"); for section in 1u64..=3 { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(section).await.expect("Failed to sync"); } drop(oversized); // Restore at the newest synced boundary. let oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("second"), cfg, (3, ENTRY_SIZE)) .await .expect("reinitialization must succeed"); // All sections should be valid for section in 1u64..=3 { let entry = oversized.get(section, 0).await.expect("Failed to get"); assert_eq!(entry.id, section); } assert_eq!(oversized.newest_section(), Some(3)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery keeps an empty index section and removes an orphan value section. #[test_traced] fn test_recovery_orphan_with_empty_index_section() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate section 1 with entries let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [1; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Manually create orphan value section 2 let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); let orphan_value: TestValue = [2; 16]; (glob, _, _) = glob .append(2, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(2).await.expect("Failed to sync glob"); drop(glob); // Now truncate index section 1 to 0 (making it empty but still tracked) let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(0).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitializing should handle empty index section and remove orphan value section let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // Section 1 should exist but have no entries (empty after truncation) assert!(oversized.get(1, 0).await.is_err()); // Orphan section 2 should be removed assert_eq!(oversized.newest_section(), Some(1)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery removes orphan value sections between and after non-contiguous index /// sections. #[test_traced] fn test_recovery_orphan_sections_with_gaps() { // Test non-contiguous sections: index has [1, 3, 5], values has [1, 2, 3, 4, 5, 6] // Orphan sections 2, 4, 6 should be removed let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create index with sections 1, 3, 5 (gaps) let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for section in [1u64, 3, 5] { let value: TestValue = [section as u8; 16]; let entry = TestEntry::new(section, 0, 0); (oversized, _, _, _) = oversized .append(section, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(section).await.expect("Failed to sync"); } drop(oversized); // Manually create orphan value sections 2, 4, 6 (filling gaps and beyond) let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); for section in [2u64, 4, 6] { let orphan_value: TestValue = [section as u8; 16]; (glob, _, _) = glob .append(section, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(section).await.expect("Failed to sync glob"); } drop(glob); // Reinitializing should remove orphan sections 2, 4, 6 let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg) .await .expect("reinitialization must succeed"); // Sections 1, 3, 5 should still be valid for section in [1u64, 3, 5] { let entry = oversized.get(section, 0).await.expect("Failed to get"); assert_eq!(entry.id, section); } // Verify only sections 1, 3, 5 exist (orphans removed) assert_eq!(oversized.oldest_section(), Some(1)); assert_eq!(oversized.newest_section(), Some(5)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Bytes appended after an index was truncated or removed cannot replace its authenticated /// prefix. #[test_traced] fn test_recovery_discards_index_extension_after_prefix_loss() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Seed two sections with one synced entry each, so both hold durable values. let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for section in 1..=2 { (oversized, _, _, _) = oversized .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .expect("Failed to append"); } // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Keep both value blobs, but erase one index by truncation and the other by removal. // Replace each with two complete pages of bytes that have no valid checksum slots. for section in 1..=2u64 { let (blob, original_size) = context .open(&cfg.index_partition, §ion.to_be_bytes()) .await .expect("Failed to open index blob"); if section == 1 { blob.resize(0).await.expect("Failed to truncate index"); blob.sync().await.expect("Failed to sync index truncation"); drop(blob); } else { drop(blob); context .remove(&cfg.index_partition, Some(§ion.to_be_bytes())) .await .expect("Failed to remove index"); } let (blob, _) = context .open(&cfg.index_partition, §ion.to_be_bytes()) .await .expect("Failed to recreate index blob"); blob.write_at( 0, vec![0; usize::try_from(original_size * 2).unwrap()], WriteOptions::SYNC, ) .await .expect("Failed to extend index"); } // Recovery must not adopt the checksum-less extensions as index data: both // sections come back empty (the original entries are gone with their prefixes) // and the truncation is durable. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("Failed to recover extended index"); for section in 1..=2u64 { assert!(matches!( oversized.get(section, 0).await, Err(Error::ItemOutOfRange(0)) )); let recovered_size = context .logical_blob(&cfg.index_partition, §ion.to_be_bytes()) .unwrap() .len(); assert_eq!(recovered_size, 0); // The recovered sections accept new entries from position zero. let position; (oversized, position, _, _) = oversized .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .expect("Failed to append after recovery"); assert_eq!(position, 0); } oversized = oversized.sync_all().await.expect("Failed to sync sentinel"); drop(oversized); // The sentinel entries written after recovery survive a clean reopen. let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), cfg) .await .expect("Failed to reopen sentinel"); for section in 1..=2u64 { let entry = oversized .get(section, 0) .await .expect("Sentinel index missing"); let (offset, size) = entry.value_location(); assert_eq!(entry.id, section); assert_eq!( oversized .get_value(section, offset, size) .await .expect("Sentinel value missing"), [section as u8; 16] ); } oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery truncates value bytes past the last entry, so the next append lands at that /// entry's end. #[test_traced] fn test_recovery_glob_trailing_garbage_truncated() { // Tests the bug fix: when value is written to glob but index entry isn't // (crash after value write, before index write), recovery should truncate // the glob trailing garbage so subsequent appends start at correct offset. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); // Append 2 entries let mut locations = Vec::new(); for i in 0..2u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); // Record where next entry SHOULD start (end of entry 1) let expected_next_offset = byte_end(locations[1].1, locations[1].2); drop(oversized); // Simulate crash: write garbage to glob (simulating partial value write) let (blob, size) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); assert_eq!(size, expected_next_offset); // Write 100 bytes of garbage (simulating partial/failed value write) let garbage = vec![0xDE; 100]; blob.write_at(size, garbage, WriteOptions::SYNC) .await .expect("Failed to write garbage"); drop(blob); // Verify glob now has trailing garbage let (blob, new_size) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); assert_eq!(new_size, expected_next_offset + 100); drop(blob); // Reinitializing should truncate the trailing garbage let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First 2 entries should still be valid for i in 0..2u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); } // A new entry should be appended at expected_next_offset, NOT at garbage end let new_value: TestValue = [99; 16]; let new_entry = TestEntry::new(99, 0, 0); let (pos, offset, _size); (oversized, pos, offset, _size) = oversized .append(1, new_entry, &new_value) .await .expect("Failed to append after recovery"); // Verify position is 2 (after the 2 existing entries) assert_eq!(pos, 2); // Verify offset is at expected_next_offset (garbage was truncated) assert_eq!(offset, expected_next_offset); // Verify we can read the new entry let retrieved = oversized.get(1, 2).await.expect("Failed to get new entry"); assert_eq!(retrieved.id, 99); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery drops an entry whose value range overflows. #[test_traced] fn test_recovery_entry_with_overflow_offset() { // Tests that an entry with offset near u64::MAX that would overflow // when added to size is detected as invalid during recovery. let executor = deterministic::Runner::default(); executor.start(|context| async move { // Use page size = entry size so one entry per page let cfg = entry_cfg(); // Create and populate with valid entry let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [1; 16]; let entry = TestEntry::new(1, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Build a corrupted entry with offset near u64::MAX that would overflow. // We need to write a valid page (with correct page-level CRC) containing // the semantically-invalid entry data. let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); // Build entry data: id (8) + value_offset (8) + value_size (4) = 20 bytes let mut entry_data = Vec::new(); 1u64.write(&mut entry_data); // id (u64::MAX - 10).write(&mut entry_data); // value_offset (near max) 100u32.write(&mut entry_data); // value_size (offset + size overflows) assert_eq!(entry_data.len(), TestEntry::SIZE); // Build page-level CRC record (12 bytes): // len1 (2) + crc1 (4) + len2 (2) + crc2 (4) let crc = Crc32::checksum(&entry_data); let len1 = TestEntry::SIZE as u16; let mut crc_record = Vec::new(); crc_record.extend_from_slice(&len1.to_be_bytes()); // len1 crc_record.extend_from_slice(&crc.to_be_bytes()); // crc1 crc_record.extend_from_slice(&0u16.to_be_bytes()); // len2 (unused) crc_record.extend_from_slice(&0u32.to_be_bytes()); // crc2 (unused) assert_eq!(crc_record.len(), 12); // Write the complete physical page: entry_data + crc_record let mut page = entry_data; page.extend_from_slice(&crc_record); blob.write_at(0, page, WriteOptions::SYNC) .await .expect("Failed to write corrupted page"); drop(blob); // Recovery on reinitialization should detect the invalid entry // (offset + size would overflow, and even with saturating_add it exceeds glob_size) let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // The corrupted entry should have been truncated (invalid) assert!(oversized.get(1, 0).await.is_err()); // Should be able to append after recovery let new_value: TestValue = [99; 16]; let new_entry = TestEntry::new(99, 0, 0); let (pos, new_offset); (oversized, pos, new_offset, _) = oversized .append(1, new_entry, &new_value) .await .expect("Failed to append after recovery"); // Position should be 0 (corrupted entry was removed) assert_eq!(pos, 0); // Offset should be 0 (glob was truncated to 0) assert_eq!(new_offset, 0); oversized.destroy().await.expect("Failed to destroy"); }); } /// A section that tracked recovery empties stays open, reuses its value offsets from zero, and /// survives a restart. #[test_traced] fn test_empty_section_persistence() { // Tests that sections that become empty (all entries removed/truncated) // are handled correctly across restart cycles. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Create and populate section 1 with entries let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); for i in 0..3u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); (oversized, _, _, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); } // Also create section 2 to ensure it survives let value2: TestValue = [10; 16]; let entry2 = TestEntry::new(10, 0, 0); (oversized, _, _, _) = oversized .append(2, entry2, &value2) .await .expect("Failed to append to section 2"); // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Truncate section 1's index to 0 (making it empty) let (blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(0).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // On the first restart, recovery should handle empty section 1 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Section 1 should exist but have no entries assert!(oversized.get(1, 0).await.is_err()); // Section 2 should still be valid let entry = oversized.get(2, 0).await.expect("Failed to get section 2"); assert_eq!(entry.id, 10); // Section 1 should still be tracked (blob exists but is empty) assert_eq!(oversized.oldest_section(), Some(1)); // Values are reachable only through index entries, so recovery removes the orphaned // bytes before the section can be reused. let new_value: TestValue = [99; 16]; let new_entry = TestEntry::new(99, 0, 0); let (pos, offset, size); (oversized, pos, offset, size) = oversized .append(1, new_entry, &new_value) .await .expect("Failed to append to empty section"); assert_eq!(pos, 0); assert_eq!(offset, 0); oversized = oversized.sync(1).await.expect("Failed to sync"); // Verify the new entry is readable after reusing the section. let entry = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(entry.id, 99); let value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(value, new_value); drop(oversized); // Verify persistence across a second restart let oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("third"), cfg.clone()) .await .expect("Failed to reinit again"); // Section 1's new entry should be valid let entry = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(entry.id, 99); // Section 2 should still be valid let entry = oversized.get(2, 0).await.expect("Failed to get section 2"); assert_eq!(entry.id, 10); oversized.destroy().await.expect("Failed to destroy"); }); } /// A value read whose size covers only the checksum fails. #[test_traced] fn test_get_value_size_equals_crc_size() { // Tests the boundary condition where size = 4 (just CRC, no data). // This should fail because there's no actual data to decode. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); let offset; (oversized, _, offset, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // Size = 4 (exactly CRC_SIZE) means 0 bytes of actual data // This should fail with ChecksumMismatch or decode error let result = oversized.get_value(1, offset, 4).await; assert!(result.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// A value read with one byte beyond the checksum fails. #[test_traced] fn test_get_value_size_just_over_crc() { // Tests size = 5 (CRC + 1 byte of data). // This should fail because the data is too short to decode. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); let value: TestValue = [42; 16]; let entry = TestEntry::new(1, 0, 0); let offset; (oversized, _, offset, _) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); // Size = 5 means 1 byte of actual data (after stripping CRC) // This should fail with checksum mismatch since we're reading wrong bytes let result = oversized.get_value(1, offset, 5).await; assert!(result.is_err()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery handles section numbers near u64::MAX. #[test_traced] fn test_recovery_maximum_section_numbers() { // Test recovery with very large section numbers near u64::MAX to check // for overflow edge cases in section arithmetic. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Use section numbers near u64::MAX let large_sections = [u64::MAX - 3, u64::MAX - 2, u64::MAX - 1]; // Create and populate with large section numbers let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for §ion in &large_sections { let value: TestValue = [(section & 0xFF) as u8; 16]; let entry = TestEntry::new(section, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(section, entry, &value) .await .expect("Failed to append"); locations.push((section, (position, offset, size))); } // One sync of every section publishes no marker, so the damage below lies above every // floor. oversized = oversized.sync_all().await.expect("Failed to sync"); drop(oversized); // Simulate crash: truncate glob for middle section let middle_section = large_sections[1]; let (blob, size) = context .open(&cfg.value_partition, &middle_section.to_be_bytes()) .await .expect("Failed to open blob"); blob.resize(size / 2).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Reinitializing should recover without overflow panics let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // First and last sections should still be valid let entry = oversized .get(large_sections[0], 0) .await .expect("Failed to get first section"); assert_eq!(entry.id, large_sections[0]); let entry = oversized .get(large_sections[2], 0) .await .expect("Failed to get last section"); assert_eq!(entry.id, large_sections[2]); // Middle section should have been truncated (no entries) assert!(oversized.get(middle_section, 0).await.is_err()); // Verify we can still append to these large sections let new_value: TestValue = [0xAB; 16]; let new_entry = TestEntry::new(999, 0, 0); (oversized, _, _, _) = oversized .append(middle_section, new_entry, &new_value) .await .expect("Failed to append after recovery"); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery completes a truncation that an earlier recovery left partially applied. #[test_traced] fn test_recovery_crash_during_recovery_truncate() { // Tests a nested crash scenario. Initial crash leaves inconsistent state, then a second // crash occurs during recovery's truncate operation. This simulates the worst-case where // recovery itself is interrupted. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Phase 1: Create valid data with 5 entries let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let mut locations = Vec::new(); for i in 0..5u8 { let value: TestValue = [i; 16]; let entry = TestEntry::new(i as u64, 0, 0); let (position, offset, size); (oversized, position, offset, size) = oversized .append(1, entry, &value) .await .expect("Failed to append"); locations.push((position, offset, size)); } oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Phase 2: Simulate first crash by truncating glob to lose last 2 entries let (blob, _) = context .open(&cfg.value_partition, &1u64.to_be_bytes()) .await .expect("Failed to open blob"); let keep_size = byte_end(locations[2].1, locations[2].2); blob.resize(keep_size).await.expect("Failed to truncate"); blob.sync().await.expect("Failed to sync"); drop(blob); // Interrupt recovery while truncating the index from five entries to three. // Keep four entries to model the intermediate state. let chunk_size = FixedJournal::::CHUNK_SIZE as u64; let (index_blob, _) = context .open(&cfg.index_partition, &1u64.to_be_bytes()) .await .expect("Failed to open index blob"); let partial_truncate_size = 4 * chunk_size; // 4 entries instead of 3 index_blob .resize(partial_truncate_size) .await .expect("Failed to resize"); index_blob.sync().await.expect("Failed to sync"); drop(index_blob); // Phase 4: Second recovery attempt should handle the inconsistent state // Index has 4 entries, but glob only supports 3. let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("Failed to reinit after nested crash"); // Only first 3 entries should be valid (recovery should truncate again) for i in 0..3u8 { let entry = oversized.get(1, i as u64).await.expect("Failed to get"); assert_eq!(entry.id, i as u64); let (_, offset, size) = locations[i as usize]; let value = oversized .get_value(1, offset, size) .await .expect("Failed to get value"); assert_eq!(value, [i; 16]); } // Entry 3 should not exist (index was truncated to match glob) assert!(oversized.get(1, 3).await.is_err()); // Verify append works after nested crash recovery let new_value: TestValue = [0xFF; 16]; let new_entry = TestEntry::new(100, 0, 0); let (pos, offset, _size); (oversized, pos, offset, _size) = oversized .append(1, new_entry, &new_value) .await .expect("Failed to append"); assert_eq!(pos, 3); // Should be position 3 (after the 3 valid entries) // Verify the offset starts where entry 2 ended (no gaps) assert_eq!(offset, byte_end(locations[2].1, locations[2].2)); oversized.destroy().await.expect("Failed to destroy"); }); } /// Tracked recovery completes an orphan value cleanup that an earlier recovery left partially /// applied. #[test_traced] fn test_recovery_crash_during_orphan_cleanup() { // Tests crash during orphan section cleanup: recovery starts removing // orphan value sections, but crashes mid-cleanup. let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); // Phase 1: Create valid data in section 1 let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("first"), cfg.clone()) .await .expect("Failed to init"); let value: TestValue = [1; 16]; let entry = TestEntry::new(1, 0, 0); let (offset1, size1); (oversized, _, offset1, size1) = oversized .append(1, entry, &value) .await .expect("Failed to append"); oversized = oversized.sync(1).await.expect("Failed to sync"); drop(oversized); // Phase 2: Create orphan value sections 2, 3, 4 (no index entries) let glob_cfg = GlobConfig { partition: cfg.value_partition.clone(), compression: cfg.compression, codec_config: (), write_buffer: cfg.value_write_buffer, }; let mut glob: Glob<_, TestValue> = GlobRecovery::init(context.child("glob"), glob_cfg) .await .expect("Failed to init glob") .finish(); for section in 2u64..=4 { let orphan_value: TestValue = [section as u8; 16]; (glob, _, _) = glob .append(section, &orphan_value) .await .expect("Failed to append orphan"); glob = glob.sync(section).await.expect("Failed to sync glob"); } drop(glob); // Phase 3: Simulate partial orphan cleanup (section 2 removed, 3 and 4 remain) // This simulates a crash during cleanup_orphan_value_sections() context .remove(&cfg.value_partition, Some(&2u64.to_be_bytes())) .await .expect("Failed to remove section 2"); // Phase 4: Recovery should complete the cleanup let mut oversized: Oversized<_, TestEntry, TestValue> = open_tracked(context.child("second"), cfg.clone()) .await .expect("reinitialization must succeed"); // Section 1 should still be valid let entry = oversized.get(1, 0).await.expect("Failed to get"); assert_eq!(entry.id, 1); let value = oversized .get_value(1, offset1, size1) .await .expect("Failed to get value"); assert_eq!(value, [1; 16]); // No orphan sections should remain assert_eq!(oversized.oldest_section(), Some(1)); assert_eq!(oversized.newest_section(), Some(1)); // Should be able to append to section 2 (now clean) let new_value: TestValue = [42; 16]; let new_entry = TestEntry::new(42, 0, 0); let pos; (oversized, pos, _, _) = oversized .append(2, new_entry, &new_value) .await .expect("Failed to append to section 2"); assert_eq!(pos, 0); // First entry in new section oversized.destroy().await.expect("Failed to destroy"); }); } /// Reading the last entry of a pruned section reports the prune. #[test_traced] fn test_last_pruned_section_returns_error() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut oversized: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context, cfg, (0, 0)) .await .expect("Failed to init"); let value: TestValue = [1; 16]; (oversized, _, _, _) = oversized .append(0, TestEntry::new(1, 0, 0), &value) .await .expect("Failed to append"); (oversized, _, _, _) = oversized .append(1, TestEntry::new(2, 0, 0), &value) .await .expect("Failed to append"); oversized = oversized.sync_all().await.expect("Failed to sync"); (oversized, _) = oversized.prune(1).await.expect("Failed to prune"); assert!(matches!( oversized.last(0).await, Err(Error::AlreadyPrunedToSection(1)) )); assert!(oversized.last(1).await.unwrap().is_some()); oversized.destroy().await.expect("Failed to destroy"); }); } /// Build the crash image of a torn index page rewrite in section 1. Four entries are synced /// (index page 0 full, page 1 a 16-byte partial tail), then five more are appended whose /// index flush rewrote page 1 as a full page and wrote a partial page 2, with one byte of /// the page-1 rewrite lost. Page 1's 16-byte durable checksum slot still validates, so /// backward tail discovery adopts page 2 while page 1 is a short interior page. The /// unsynced values are lost. async fn torn_rewrite_image(context: deterministic::Context) { let mut journal = open_tracked(context.child("seed"), test_cfg()) .await .unwrap(); for id in 0..4u64 { (journal, _, _, _) = journal .append(1, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } journal = journal.sync(1).await.unwrap(); // An empty flush publishes the section's marker. journal = journal.sync(Vec::::new()).await.unwrap(); for id in 4..9u64 { (journal, _, _, _) = journal .append(1, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } journal.index = journal.index.sync(1).await.unwrap(); drop(journal); // Lose one byte of the page-1 rewrite beyond its durable 16 bytes. let physical_page = 64 + commonware_runtime::buffer::paged::CHECKSUM_SIZE; let (blob, _) = context .open("test-index", &1u64.to_be_bytes()) .await .unwrap(); blob.write_at(physical_page + 16 + 7, vec![0u8], WriteOptions::SYNC) .await .unwrap(); } /// A tracked open keeps every synced entry below a torn index page rewrite and drops the /// entries the torn page hides. #[test_traced] fn test_oversized_tracked_open_keeps_synced_entries_below_torn_rewrite() { let chunk = ENTRY_SIZE; let (_, checkpoint) = deterministic::Runner::default().start_and_recover(torn_rewrite_image); deterministic::Runner::from(checkpoint).start(move |context| async move { // Recovery keeps the four synced entries, and the last one reads back its value. let journal = open_tracked(context.child("reopen"), test_cfg()) .await .unwrap(); assert_eq!(journal.size(1).unwrap(), 4 * chunk); let entry = journal.get(1, 3).await.unwrap(); assert_eq!( journal .get_value(1, entry.value_offset, entry.value_size) .await .unwrap(), [3; 16] ); }); } /// A tracked journal reopens after a destroy that stopped inside its marker store. The /// marker generation written by a prune is the newest, and an interrupted destroy never /// leaves an older generation that names the pruned section. #[test_traced] fn test_oversized_interrupted_destroy_after_prune_reopens() { const MARKERS: &str = "destroy-markers"; let mut found = None; for seed in 0..64u64 { let (interrupted, checkpoint) = deterministic::Runner::seeded(seed).start_and_recover(|context| async move { let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("first"), test_cfg(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); while let Some(item) = replay.next().await { item.unwrap(); } let mut journal = replay.finish().await.unwrap(); for section in 0..2u64 { (journal, _, _, _) = journal .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .unwrap(); } // Publish both markers, then prune section 0, which durably removes its // marker as a newer generation before deleting its blobs. journal = journal.sync_all().await.unwrap(); journal = journal.sync(Vec::::new()).await.unwrap(); (journal, _) = journal.prune(1).await.unwrap(); // Fail removals at random until one destroy stops with one marker copy left. *context.storage_fault_config().write() = deterministic::FaultConfig { remove_rate: Some(probability!(0.5)), ..Default::default() }; let failed = journal.destroy().await.is_err(); *context.storage_fault_config().write() = deterministic::FaultConfig::default(); failed && context .scan(MARKERS) .await .is_ok_and(|names| names.len() == 1) }); if interrupted { found = Some(checkpoint); break; } } let checkpoint = found.expect("some seed interrupts destroy with one marker copy left"); deterministic::Runner::from(checkpoint).start(|context| async move { let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("second"), test_cfg(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); while let Some(item) = replay.next().await { item.unwrap(); } let journal = replay.finish().await.unwrap(); assert_eq!(journal.get(1, 0).await.unwrap().id, 1); }); } /// A tracked prune interrupted at any step reopens to every entry at or above the prune floor, /// with each section below it retained whole or removed. #[test_traced] fn test_oversized_interrupted_prune_reopens_to_consistent_prefix() { const MARKERS: &str = "prune-markers"; const SECTIONS: u64 = 3; const ENTRIES: u64 = 2; const FLOOR: u64 = 2; // Stop the prune before its first mutation, then its second, and so on until one prune // completes, and crash after each. let mut interrupted = 0; loop { let faults = MutationFaults::default(); let (pruned, checkpoint) = deterministic::Runner::default().start_and_recover({ let faults = faults.clone(); |context| async move { let context = MutationFaultContext { inner: context, faults: faults.clone(), }; // Open an empty tracked journal. let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("first"), test_cfg(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); while let Some(item) = replay.next().await { item.unwrap(); } let mut journal = replay.finish().await.unwrap(); // Make every section durable and publish a marker for each. for section in 0..SECTIONS { for position in 0..ENTRIES { let id = section * ENTRIES + position; (journal, _, _, _) = journal .append(section, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); } } journal = journal.sync_all().await.unwrap(); journal = journal.sync(Vec::::new()).await.unwrap(); faults.allow(interrupted); journal.prune(FLOOR).await.is_ok() } }); let stopped = faults.failed(); let scenario = format!("stopped after {interrupted} mutations"); assert_eq!(pruned, !stopped, "{scenario}"); // Sections at or above the floor are whole, and each earlier section is whole or gone. deterministic::Runner::from(checkpoint).start(|context| async move { let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("second"), test_cfg(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap_or_else(|err| panic!("{scenario}: must reopen: {err}")); while let Some(item) = replay.next().await { item.unwrap(); } let journal = replay.finish().await.unwrap(); for section in 0..SECTIONS { let retained = match journal.size(section) { Ok(size) => size / ENTRY_SIZE, Err(Error::SectionOutOfRange(_)) => 0, Err(err) => panic!("section {section} must report its size: {err:?}"), }; assert!( retained == ENTRIES || (section < FLOOR && retained == 0), "{scenario}: section {section} must be whole or pruned, retained {retained}" ); // Nothing is removed before the first mutation, and no section below the // floor survives a completed prune. if interrupted == 0 { assert_eq!(retained, ENTRIES, "{scenario}: section {section}"); } if pruned && section < FLOOR { assert_eq!(retained, 0, "{scenario}: section {section}"); } for position in 0..retained { let entry = journal.get(section, position).await.unwrap(); assert_eq!( entry.id, section * ENTRIES + position, "{scenario}: a retained entry must keep its position" ); let (offset, size) = entry.value_location(); assert_eq!( journal.get_value(section, offset, size).await.unwrap(), [u8::try_from(entry.id).expect("id must fit in a byte"); 16], "{scenario}: a retained entry must read back its value" ); } } }); if !stopped { break; } interrupted += 1; } // Prune rewrites the markers, then removes the index section and the value section of // every section below the floor, so it can stop at least once before each. assert!(interrupted > 2 * FLOOR, "{interrupted}"); } /// A checkpoint restore after a prune interrupted at any step keeps every entry at or above /// the oldest retained index section. The index is pruned before the values, so an /// interruption can leave value sections with no index section, which the restore removes, /// and never an index section whose values are gone. #[test_traced] fn test_init_with_checkpoint_after_interrupted_prune() { const SECTIONS: u64 = 4; const FLOOR: u64 = 3; // Stop the prune before its first mutation, then its second, and so on until one prune // completes, and crash after each. let mut orphaned = 0; let mut interrupted = 0; loop { let faults = MutationFaults::default(); let (pruned, checkpoint) = deterministic::Runner::default().start_and_recover({ let faults = faults.clone(); |context| async move { let context = MutationFaultContext { inner: context, faults: faults.clone(), }; // Make one entry per section durable. let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), test_cfg(), (0, 0)) .await .unwrap(); for section in 0..SECTIONS { (journal, _, _, _) = journal .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .unwrap(); } journal = journal.sync_all().await.unwrap(); faults.allow(interrupted); journal.prune(FLOOR).await.is_ok() } }); let stopped = faults.failed(); let scenario = format!("stopped after {interrupted} mutations"); assert_eq!(pruned, !stopped, "{scenario}"); let orphans = deterministic::Runner::from(checkpoint).start(|context| async move { // A prune that stopped after an index removal leaves value sections with no // index section, and no stop leaves an index section without its values. let index = context.scan("test-index").await.unwrap(); let values = context.scan("test-values").await.unwrap(); let orphans = values.iter().any(|section| !index.contains(section)); assert!( index.iter().all(|section| values.contains(section)), "{scenario}: every index section must keep its value section" ); // The restore keeps the last section whole and every section the index retains. let journal = Oversized::<_, TestEntry, TestValue>::init_with_checkpoint( context.child("restore"), test_cfg(), (SECTIONS - 1, ENTRY_SIZE), ) .await .unwrap_or_else(|err| panic!("{scenario}: must restore: {err}")); let oldest = journal .oldest_section() .expect("the last section is retained"); assert!( oldest <= FLOOR, "{scenario}: the prune floor bounds what a prune removes" ); for section in oldest..SECTIONS { let entry = journal.get(section, 0).await.unwrap(); assert_eq!( entry.id, section, "{scenario}: a retained entry must keep its section" ); assert_eq!( journal .get_value(section, entry.value_offset, entry.value_size) .await .unwrap(), [u8::try_from(section).expect("section must fit in a byte"); 16], "{scenario}: a retained entry must read back its value" ); } // Nothing is removed before the first mutation, and no section below the // floor survives a completed prune. if interrupted == 0 { assert_eq!(oldest, 0, "{scenario}"); } if pruned { assert_eq!(oldest, FLOOR, "{scenario}"); } // The value sections that lost their index section are gone. assert_eq!( context.scan("test-values").await.unwrap(), context.scan("test-index").await.unwrap(), "{scenario}: every value section must have an index section" ); journal.destroy().await.unwrap(); orphans }); if orphans { orphaned += 1; } if !stopped { break; } interrupted += 1; } // Prune removes the index section and then the value section of every section below the // floor, so it can stop at least once before each. The value sections outlive the index // removals, so every stop before a value removal leaves one with no index section. assert!(interrupted >= 2 * FLOOR, "{interrupted}"); assert!(orphaned >= FLOOR, "{orphaned}"); } /// A checkpoint restore still rejects a value-only section between the oldest retained index /// section and the checkpoint, since only lost index data leaves that gap. #[test_traced] fn test_init_with_checkpoint_rejects_value_only_gap() { let executor = deterministic::Runner::default(); executor.start(|context| async move { let cfg = test_cfg(); let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("first"), cfg.clone(), (0, 0)) .await .unwrap(); for section in 0..3u64 { (journal, _, _, _) = journal .append(section, TestEntry::new(section, 0, 0), &[section as u8; 16]) .await .unwrap(); } journal = journal.sync_all().await.unwrap(); let checkpoint = (2, journal.size(2).unwrap()); drop(journal); // Lose index section 1 while its values survive. The gap lies above the oldest // retained index section, which no interrupted prune can produce. context .remove(&cfg.index_partition, Some(&1u64.to_be_bytes())) .await .unwrap(); let result = Oversized::<_, TestEntry, TestValue>::init_with_checkpoint( context.child("restore"), cfg.clone(), checkpoint, ) .await; assert!(matches!(result, Err(Error::Corruption(_)))); assert_eq!(context.scan(&cfg.value_partition).await.unwrap().len(), 3); }); } /// Restoring the empty checkpoint on empty storage opens a fresh journal without reading, /// writing, syncing, or creating any blob, and that journal starts a history a later restore /// reopens. #[test_traced] fn test_init_with_checkpoint_empty_opens_nothing() { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Record every read, write, and durability operation of the restore. let cfg = test_cfg(); let (recording, recordings) = commonware_runtime::mocks::RecordingContext::new(context.child("restore")); let pending = PendingSyncs::default(); pending.arm(); let delayed = DelayedSyncContext { inner: recording, pending: pending.clone(), }; let journal = drive_pending_syncs( &pending, Oversized::<_, TestEntry, TestValue>::init_with_checkpoint( delayed, cfg.clone(), (0, 0), ), ) .await .unwrap(); assert_eq!(journal.oldest_section(), None); assert_eq!(pending.calls(), 0, "an empty restore must not sync"); let observed = recordings.snapshot(); assert!(observed.reads.is_empty(), "an empty restore must not read"); assert!( observed.writes.is_empty(), "an empty restore must not write" ); // Neither partition exists, so the restore opened no blob. for partition in [&cfg.index_partition, &cfg.value_partition] { assert!( matches!( context.scan(partition).await, Err(RError::PartitionMissing(_)) ), "an empty restore must not create {partition}" ); } // The fresh journal accepts an append that a restore at its synced boundary reopens. pending.unblock(); let (journal, position, offset, size) = journal .append(1, TestEntry::new(7, 0, 0), &[7; 16]) .await .unwrap(); assert_eq!(position, 0); drop(journal.sync(1).await.unwrap()); let journal = Oversized::<_, TestEntry, TestValue>::init_with_checkpoint( context.child("reopen"), cfg, (1, ENTRY_SIZE), ) .await .unwrap(); assert_eq!(journal.get(1, 0).await.unwrap().id, 7); assert_eq!(journal.get_value(1, offset, size).await.unwrap(), [7; 16]); journal.destroy().await.unwrap(); }); } /// Cancelling value validation cannot skip an entry or publish a marker beyond it. #[rstest::rstest] #[case::valid_finish(false, false)] #[case::valid_next(false, true)] #[case::corrupt_finish(true, false)] #[case::corrupt_next(true, true)] fn test_oversized_replay_dropped_during_value_read_fails_replay( #[case] corrupt: bool, #[case] resume: bool, ) { deterministic::Runner::default().start(|context| async move { const SECTION: u64 = 1; const MARKERS: &str = "cancel-value-markers"; let cfg = test_cfg(); let mut journal: Oversized<_, TestEntry, TestValue> = Oversized::init_with_checkpoint(context.child("seed"), cfg.clone(), (0, 0)) .await .unwrap(); let mut second_offset = 0; for id in 0..3 { let offset; (journal, _, offset, _) = journal .append(SECTION, TestEntry::new(id, 0, 0), &[id as u8; 16]) .await .unwrap(); if id == 1 { second_offset = offset; } } drop(journal.sync_all().await.unwrap()); // Only the first entry is committed. The remaining values require verification. let mut markers: Metadata<_, SectionKey, u64> = Metadata::init( context.child("seed_markers"), MetadataConfig { partition: MARKERS.into(), codec_config: (), }, ) .await .unwrap(); markers.put(SectionKey::new(SECTION), 1); drop(markers.sync().await.unwrap()); if corrupt { let (blob, _) = context .open(&cfg.value_partition, &SECTION.to_be_bytes()) .await .unwrap(); blob.write_at(second_offset, vec![0xff], WriteOptions::SYNC) .await .unwrap(); } let (gated, gate) = ReadGateContext::new(context.child("cancel"), cfg.value_partition.clone()); let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( gated, cfg.clone(), MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); // Dropping a call before it is polled starts no read and leaves the replay usable. drop(replay.next()); // Warm the index while yielding the committed entry, then stop only a value read. assert_eq!(replay.next().await.unwrap().unwrap().2.id, 0); gate.arm(); { let next = replay.next(); futures::pin_mut!(next); assert!(futures::poll!(next.as_mut()).is_pending()); assert_eq!(gate.blocked(), 1); } gate.disarm(); if resume { assert!(matches!( replay.next().await, Some(Err(Error::ReplayInterrupted)) )); assert!(replay.next().await.is_none()); assert!(replay.next().await.is_none()); } assert!(matches!(replay.finish().await, Err(Error::ReplayFailed))); let markers: Metadata<_, SectionKey, u64> = Metadata::init( context.child("check_markers"), MetadataConfig { partition: MARKERS.into(), codec_config: (), }, ) .await .unwrap(); assert_eq!(markers.get(&SectionKey::new(SECTION)), Some(&1)); drop(markers); // Recovery rechecks the interrupted entry and retains only the valid prefix. let mut replay = Oversized::<_, TestEntry, TestValue>::init_with_markers( context.child("reopen"), cfg, MARKERS.into(), ReadOptions::default(), ) .await .unwrap(); let retained = if corrupt { 1 } else { 3 }; for id in 0..retained { let (section, position, entry) = replay.next().await.unwrap().unwrap(); assert_eq!((section, position, entry.id), (SECTION, id, id)); } assert!(replay.next().await.is_none()); let journal = replay.finish().await.unwrap(); assert_eq!(journal.size(SECTION).unwrap(), retained * ENTRY_SIZE); assert_eq!( journal .tracking .as_ref() .unwrap() .metadata .get(&SectionKey::new(SECTION)), Some(&retained) ); for id in 0..retained { let entry = journal.get(SECTION, id).await.unwrap(); assert_eq!( journal .get_value(SECTION, entry.value_offset, entry.value_size) .await .unwrap(), [id as u8; 16] ); } journal.destroy().await.unwrap(); }); } }