From c31d68d949ca2719fd0f80d8e9caf7112f9f8a0d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 14 Mar 2026 15:26:43 +0200 Subject: [PATCH 01/14] feat: support custom sequence number generators - Accept custom SequenceNumberGenerator via DatabaseBuilder - Thread SharedSequenceNumberGenerator through Supervisor, MetaKeyspace, SnapshotTracker, and recovery - Default to SequenceNumberCounter when no custom generator is set - Add integration tests: custom generator, recovery, default behavior Closes fjall-rs/fjall#174 --- src/batch/mod.rs | 2 +- src/builder.rs | 28 ++++++- src/db.rs | 30 ++++--- src/db_config.rs | 9 ++- src/keyspace/mod.rs | 4 +- src/lib.rs | 3 +- src/meta_keyspace.rs | 12 +-- src/recovery.rs | 4 +- src/snapshot_tracker.rs | 23 +++--- src/supervisor.rs | 4 +- tests/custom_seqno_generator.rs | 138 ++++++++++++++++++++++++++++++++ 11 files changed, 221 insertions(+), 36 deletions(-) create mode 100644 tests/custom_seqno_generator.rs diff --git a/src/batch/mod.rs b/src/batch/mod.rs index 14011e5a..21f2d107 100644 --- a/src/batch/mod.rs +++ b/src/batch/mod.rs @@ -6,7 +6,7 @@ pub mod item; use crate::{Database, Keyspace, PersistMode}; use item::Item; -use lsm_tree::{AbstractTree, UserKey, UserValue, ValueType}; +use lsm_tree::{AbstractTree, SequenceNumberGenerator, UserKey, UserValue, ValueType}; use std::collections::HashSet; /// An atomic write batch diff --git a/src/builder.rs b/src/builder.rs index 6fc7a7cf..27a312b0 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -3,7 +3,7 @@ // (found in the LICENSE-* files in the repository) use crate::{db_config::CompactionFilterAssigner, tx::single_writer::Openable, Config}; -use lsm_tree::{Cache, CompressionType, DescriptorTable}; +use lsm_tree::{Cache, CompressionType, DescriptorTable, SharedSequenceNumberGenerator}; use std::{marker::PhantomData, path::Path, sync::Arc}; /// Database builder @@ -187,4 +187,30 @@ impl Builder { self.inner.compaction_filter_factory_assigner = Some(f); self } + + /// Sets a custom sequence number generator. + /// + /// By default, the database uses [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter), + /// a simple atomic counter. Use this to plug in a custom generator, + /// e.g., a Hybrid Logical Clock (HLC) for distributed databases. + /// + /// The generator is shared across all keyspaces and is used for + /// both write sequencing and MVCC visibility. + /// + /// # Examples + /// + /// ``` + /// use fjall::Database; + /// use std::sync::Arc; + /// + /// # let folder = tempfile::tempdir()?; + /// // Use default generator (SequenceNumberCounter) + /// let db = Database::builder(&folder).open()?; + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn seqno_generator(mut self, generator: SharedSequenceNumberGenerator) -> Self { + self.inner.seqno_generator = Some(generator); + self + } } diff --git a/src/db.rs b/src/db.rs index 279bcb53..3c06552a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -23,7 +23,9 @@ use crate::{ write_buffer_manager::WriteBufferManager, HashMap, Keyspace, KeyspaceCreateOptions, }; -use lsm_tree::{AbstractTree, SequenceNumberCounter}; +use lsm_tree::{ + AbstractTree, SequenceNumberCounter, SequenceNumberGenerator, SharedSequenceNumberGenerator, +}; use std::{ fs::remove_dir_all, path::Path, @@ -56,7 +58,7 @@ pub struct DatabaseInner { pub(crate) stats: Arc, - pub(crate) keyspace_id_counter: SequenceNumberCounter, + pub(crate) keyspace_id_counter: SharedSequenceNumberGenerator, pub worker_pool: WorkerPool, @@ -593,15 +595,19 @@ impl Database { let journal_manager = JournalManager::new(); - let seqno = SequenceNumberCounter::default(); - let visible_seqno = SequenceNumberCounter::default(); + let seqno: SharedSequenceNumberGenerator = config + .seqno_generator + .clone() + .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); + let visible_seqno: SharedSequenceNumberGenerator = + Arc::new(SequenceNumberCounter::default()); let keyspaces = Arc::new(RwLock::new(Keyspaces::with_capacity_and_hasher( 10, xxhash_rust::xxh3::Xxh3Builder::new(), ))); - let meta_tree = lsm_tree::Config::new( + let meta_tree = lsm_tree::Config::new_with_generators( config.path.join(KEYSPACES_FOLDER).join("0"), seqno.clone(), visible_seqno.clone(), @@ -652,7 +658,7 @@ impl Database { let inner = DatabaseInner { supervisor, worker_pool: WorkerPool::prepare(), - keyspace_id_counter: SequenceNumberCounter::new(1), + keyspace_id_counter: Arc::new(SequenceNumberCounter::new(1)), meta_keyspace: meta_keyspace.clone(), config, stop_signal: lsm_tree::stop_signal::StopSignal::default(), @@ -834,15 +840,19 @@ impl Database { fsync_directory(&keyspaces_folder_path)?; fsync_directory(&config.path)?; - let seqno = SequenceNumberCounter::default(); - let visible_seqno = SequenceNumberCounter::default(); + let seqno: SharedSequenceNumberGenerator = config + .seqno_generator + .clone() + .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); + let visible_seqno: SharedSequenceNumberGenerator = + Arc::new(SequenceNumberCounter::default()); let keyspaces = Arc::new(RwLock::new(Keyspaces::with_capacity_and_hasher( 10, xxhash_rust::xxh3::Xxh3Builder::new(), ))); - let meta_tree = lsm_tree::Config::new( + let meta_tree = lsm_tree::Config::new_with_generators( config.path.join(KEYSPACES_FOLDER).join("0"), seqno.clone(), visible_seqno.clone(), @@ -892,7 +902,7 @@ impl Database { let inner = DatabaseInner { supervisor, worker_pool: WorkerPool::prepare(), - keyspace_id_counter: SequenceNumberCounter::new(1), + keyspace_id_counter: Arc::new(SequenceNumberCounter::new(1)), meta_keyspace, config, stop_signal: lsm_tree::stop_signal::StopSignal::default(), diff --git a/src/db_config.rs b/src/db_config.rs index 4531ef85..f9991990 100644 --- a/src/db_config.rs +++ b/src/db_config.rs @@ -3,7 +3,7 @@ // (found in the LICENSE-* files in the repository) use crate::path::absolute_path; -use lsm_tree::{Cache, CompressionType, DescriptorTable}; +use lsm_tree::{Cache, CompressionType, DescriptorTable, SharedSequenceNumberGenerator}; use std::{ path::{Path, PathBuf}, sync::Arc, @@ -48,6 +48,12 @@ pub struct Config { // pub(crate) journal_recovery_mode: RecoveryMode, // pub(crate) compaction_filter_factory_assigner: Option, + + /// Custom sequence number generator. + /// + /// When set, this generator is used instead of the default + /// [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter). + pub(crate) seqno_generator: Option, } const DEFAULT_CPU_CORES: usize = 4; @@ -90,6 +96,7 @@ impl Config { cache: Arc::new(Cache::with_capacity_bytes(/* 32 MiB */ 32 * 1_024 * 1_024)), compaction_filter_factory_assigner: None, + seqno_generator: None, } } } diff --git a/src/keyspace/mod.rs b/src/keyspace/mod.rs index 64aba2ac..e8cbe347 100644 --- a/src/keyspace/mod.rs +++ b/src/keyspace/mod.rs @@ -20,7 +20,7 @@ use crate::{ worker_pool::WorkerMessage, Database, Guard, Iter, }; -use lsm_tree::{AbstractTree, AnyTree, SeqNo, UserKey, UserValue}; +use lsm_tree::{AbstractTree, AnyTree, SeqNo, SequenceNumberGenerator, UserKey, UserValue}; use options::CreateOptions; use std::{ ops::RangeBounds, @@ -334,7 +334,7 @@ impl Keyspace { std::fs::create_dir_all(&base_folder)?; - let base_config = lsm_tree::Config::new( + let base_config = lsm_tree::Config::new_with_generators( base_folder, db.supervisor.seqno.clone(), db.supervisor.snapshot_tracker.get_ref(), diff --git a/src/lib.rs b/src/lib.rs index f04ffa8b..39fa47c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -170,7 +170,8 @@ pub use tx::optimistic::{ pub use lsm_tree::{AbstractTree, AnyTree, Error as LsmError, TreeType}; pub use lsm_tree::{ - CompressionType, KvPair, KvSeparationOptions, SeqNo, Slice, UserKey, UserValue, + CompressionType, KvPair, KvSeparationOptions, SeqNo, SequenceNumberCounter, + SequenceNumberGenerator, SharedSequenceNumberGenerator, Slice, UserKey, UserValue, }; /// Utility functions diff --git a/src/meta_keyspace.rs b/src/meta_keyspace.rs index 56702528..3da5632f 100644 --- a/src/meta_keyspace.rs +++ b/src/meta_keyspace.rs @@ -4,7 +4,9 @@ use crate::{db::Keyspaces, keyspace::InternalKeyspaceId, Keyspace}; use byteview::StrView; -use lsm_tree::{AbstractTree, AnyTree, SeqNo, SequenceNumberCounter, UserValue}; +use lsm_tree::{ + AbstractTree, AnyTree, SeqNo, SequenceNumberGenerator, SharedSequenceNumberGenerator, UserValue, +}; use std::sync::{Arc, RwLock, RwLockWriteGuard}; pub fn encode_config_key( @@ -30,16 +32,16 @@ pub struct MetaKeyspace { #[doc(hidden)] pub keyspaces: Arc>, - seqno_generator: SequenceNumberCounter, - visible_seqno: SequenceNumberCounter, + seqno_generator: SharedSequenceNumberGenerator, + visible_seqno: SharedSequenceNumberGenerator, } impl MetaKeyspace { pub(crate) fn new( inner: AnyTree, keyspaces: Arc>, - seqno_generator: SequenceNumberCounter, - visible_seqno: SequenceNumberCounter, + seqno_generator: SharedSequenceNumberGenerator, + visible_seqno: SharedSequenceNumberGenerator, ) -> Self { Self { inner, diff --git a/src/recovery.rs b/src/recovery.rs index 96410247..070a682a 100644 --- a/src/recovery.rs +++ b/src/recovery.rs @@ -13,7 +13,7 @@ use crate::{ meta_keyspace::MetaKeyspace, Database, HashMap, Keyspace, }; -use lsm_tree::AbstractTree; +use lsm_tree::{AbstractTree, SequenceNumberGenerator}; use std::{path::PathBuf, sync::Arc}; /// Recovers keyspaces @@ -85,7 +85,7 @@ pub fn recover_keyspaces(db: &Database, meta_keyspace: &MetaKeyspace) -> crate:: recovered_config = recovered_config.with_compaction_filter_factory(f); } - let base_config = lsm_tree::Config::new( + let base_config = lsm_tree::Config::new_with_generators( path, db.supervisor.seqno.clone(), db.supervisor.snapshot_tracker.get_ref(), diff --git a/src/snapshot_tracker.rs b/src/snapshot_tracker.rs index 3d68fec2..8b23510a 100644 --- a/src/snapshot_tracker.rs +++ b/src/snapshot_tracker.rs @@ -4,12 +4,12 @@ use crate::{snapshot_nonce::SnapshotNonce, SeqNo}; use dashmap::DashMap; -use lsm_tree::SequenceNumberCounter; +use lsm_tree::{SequenceNumberGenerator, SharedSequenceNumberGenerator}; use std::sync::{atomic::AtomicU64, Arc, RwLock}; /// Keeps track of open snapshots pub struct SnapshotTrackerInner { - seqno: SequenceNumberCounter, + seqno: SharedSequenceNumberGenerator, gc_lock: RwLock<()>, @@ -33,7 +33,7 @@ impl std::ops::Deref for SnapshotTracker { } impl SnapshotTracker { - pub fn new(seqno: SequenceNumberCounter) -> Self { + pub fn new(seqno: SharedSequenceNumberGenerator) -> Self { Self(Arc::new(SnapshotTrackerInner { data: DashMap::default(), freed_count: AtomicU64::default(), @@ -43,7 +43,7 @@ impl SnapshotTracker { })) } - pub fn get_ref(&self) -> SequenceNumberCounter { + pub fn get_ref(&self) -> SharedSequenceNumberGenerator { self.seqno.clone() } @@ -183,6 +183,7 @@ impl SnapshotTracker { #[expect(clippy::unwrap_used)] mod tests { use super::*; + use lsm_tree::SequenceNumberCounter; use test_log::test; #[test] @@ -285,7 +286,7 @@ mod tests { fn snapshot_tracker_basic() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno.clone()); + let map = SnapshotTracker::new(global_seqno.clone().into()); let nonce = map.open(); assert_eq!(0, nonce.instant); @@ -304,7 +305,7 @@ mod tests { fn snapshot_tracker_increase_watermark() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno.clone()); + let map = SnapshotTracker::new(global_seqno.clone().into()); // Simulates some tx committing for _ in 0..100_000 { @@ -320,7 +321,7 @@ mod tests { fn snapshot_tracker_prevent_watermark() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno.clone()); + let map = SnapshotTracker::new(global_seqno.clone().into()); // This nonce prevents the watermark from increasing let _old_nonce = map.open(); @@ -342,7 +343,7 @@ mod tests { #[test] fn snapshot_tracker_close_never_opened_does_not_underflow_or_panic() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno); + let map = SnapshotTracker::new(global_seqno.into()); assert_eq!(map.len(), 0); map.close_raw(42); @@ -352,7 +353,7 @@ mod tests { #[test] fn snapshot_tracker_concurrent_open_same_seqno_counts_correctly() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno); + let map = SnapshotTracker::new(global_seqno.into()); // make sure seqno doesn't change between two opens let n1 = map.open(); @@ -372,7 +373,7 @@ mod tests { #[test] fn snapshot_tracker_publish_moves_seqno_forward_and_ignores_older() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno); + let map = SnapshotTracker::new(global_seqno.into()); let big = 100u64; map.publish(big); @@ -386,7 +387,7 @@ mod tests { #[test] fn snapshot_tracker_clone_snapshot_behaves_like_second_open() { let global_seqno = SequenceNumberCounter::default(); - let map = SnapshotTracker::new(global_seqno); + let map = SnapshotTracker::new(global_seqno.into()); let orig = map.open(); let clone = map.clone_snapshot(&orig); diff --git a/src/supervisor.rs b/src/supervisor.rs index 92153724..4ad8ff79 100644 --- a/src/supervisor.rs +++ b/src/supervisor.rs @@ -2,7 +2,7 @@ // This source code is licensed under both the Apache 2.0 and MIT License // (found in the LICENSE-* files in the repository) -use lsm_tree::SequenceNumberCounter; +use lsm_tree::SharedSequenceNumberGenerator; use crate::{ db::Keyspaces, @@ -23,7 +23,7 @@ pub struct SupervisorInner { pub(crate) write_buffer_size: WriteBufferManager, pub(crate) flush_manager: FlushManager, - pub seqno: SequenceNumberCounter, + pub seqno: SharedSequenceNumberGenerator, pub snapshot_tracker: SnapshotTracker, diff --git a/tests/custom_seqno_generator.rs b/tests/custom_seqno_generator.rs new file mode 100644 index 00000000..f82d1a3d --- /dev/null +++ b/tests/custom_seqno_generator.rs @@ -0,0 +1,138 @@ +use fjall::{Database, KeyspaceCreateOptions, SeqNo, SequenceNumberGenerator}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +/// A tracking generator that records how many times each method was called. +#[derive(Debug)] +struct TrackingGenerator { + inner: AtomicU64, + next_count: AtomicU64, + get_count: AtomicU64, + set_count: AtomicU64, + fetch_max_count: AtomicU64, +} + +impl TrackingGenerator { + fn new() -> Self { + Self { + inner: AtomicU64::new(0), + next_count: AtomicU64::new(0), + get_count: AtomicU64::new(0), + set_count: AtomicU64::new(0), + fetch_max_count: AtomicU64::new(0), + } + } +} + +impl SequenceNumberGenerator for TrackingGenerator { + fn next(&self) -> SeqNo { + self.next_count.fetch_add(1, Ordering::Relaxed); + self.inner.fetch_add(1, Ordering::AcqRel) + } + + fn get(&self) -> SeqNo { + self.get_count.fetch_add(1, Ordering::Relaxed); + self.inner.load(Ordering::Acquire) + } + + fn set(&self, value: SeqNo) { + self.set_count.fetch_add(1, Ordering::Relaxed); + self.inner.store(value, Ordering::Release); + } + + fn fetch_max(&self, value: SeqNo) { + self.fetch_max_count.fetch_add(1, Ordering::Relaxed); + self.inner.fetch_max(value, Ordering::AcqRel); + } +} + +#[test] +fn custom_generator_basic() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let generator = Arc::new(TrackingGenerator::new()); + let db = Database::builder(&folder) + .seqno_generator(generator.clone()) + .open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + tree.insert("a", "hello")?; + tree.insert("b", "world")?; + + assert!( + generator.next_count.load(Ordering::Relaxed) >= 2, + "Custom generator should be called for write operations", + ); + + assert_eq!(b"hello", &*tree.get("a")?.unwrap()); + assert_eq!(b"world", &*tree.get("b")?.unwrap()); + + Ok(()) +} + +#[test] +fn custom_generator_recovery() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + // Write with custom generator + { + let generator = Arc::new(TrackingGenerator::new()); + let db = Database::builder(&folder) + .seqno_generator(generator.clone()) + .open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + tree.insert("key1", "value1")?; + tree.insert("key2", "value2")?; + + db.persist(fjall::PersistMode::SyncAll)?; + } + + // Recover with a new custom generator — recovery calls .set()/.fetch_max() + { + let generator = Arc::new(TrackingGenerator::new()); + let db = Database::builder(&folder) + .seqno_generator(generator.clone()) + .open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + // Data survived recovery + assert_eq!(b"value1", &*tree.get("key1")?.unwrap()); + assert_eq!(b"value2", &*tree.get("key2")?.unwrap()); + + // Recovery used the generator (fetch_max or get) + let fetch_max_calls = generator.fetch_max_count.load(Ordering::Relaxed); + let get_calls = generator.get_count.load(Ordering::Relaxed); + assert!( + fetch_max_calls > 0 || get_calls > 0, + "Recovery should use the custom generator", + ); + + // Can write after recovery + tree.insert("key3", "value3")?; + assert_eq!(b"value3", &*tree.get("key3")?.unwrap()); + } + + Ok(()) +} + +#[test] +fn default_generator_unchanged() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + // Without custom generator, everything works as before + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + tree.insert("a", "1")?; + tree.insert("b", "2")?; + + assert_eq!(b"1", &*tree.get("a")?.unwrap()); + assert_eq!(b"2", &*tree.get("b")?.unwrap()); + + Ok(()) +} From 205b0b785ad3113245d7eddfe25ef8b7f153b124 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 14 Mar 2026 18:05:21 +0200 Subject: [PATCH 02/14] docs(builder): clarify seqno_generator affects write sequencing only The custom generator is used for write sequencing, not for MVCC visibility. The visibility watermark is managed internally by the snapshot tracker using a separate counter. --- src/builder.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 27a312b0..0f661b1e 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -188,14 +188,15 @@ impl Builder { self } - /// Sets a custom sequence number generator. + /// Sets a custom sequence number generator for write operations. /// /// By default, the database uses [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter), /// a simple atomic counter. Use this to plug in a custom generator, /// e.g., a Hybrid Logical Clock (HLC) for distributed databases. /// /// The generator is shared across all keyspaces and is used for - /// both write sequencing and MVCC visibility. + /// write sequencing (assigning seqnos to new writes). The MVCC + /// visibility watermark is managed internally by the snapshot tracker. /// /// # Examples /// From 376164a40eb97eda41c8d1afddf7187584492dcd Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 14 Mar 2026 18:46:16 +0200 Subject: [PATCH 03/14] docs(db): explain why visible_seqno uses separate default counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit visible_seqno is an MVCC visibility watermark updated via publish()/set(), not a sequence generator — intentionally decoupled from the custom write seqno generator. --- src/db.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/db.rs b/src/db.rs index 3c06552a..183f37af 100644 --- a/src/db.rs +++ b/src/db.rs @@ -599,6 +599,11 @@ impl Database { .seqno_generator .clone() .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); + + // NOTE: visible_seqno is intentionally a separate default counter, not the custom + // generator. It serves as the MVCC visibility watermark, updated via publish()/set() + // from the write seqno values — it tracks "up to which seqno are writes visible", + // not "generate the next seqno". let visible_seqno: SharedSequenceNumberGenerator = Arc::new(SequenceNumberCounter::default()); @@ -844,6 +849,11 @@ impl Database { .seqno_generator .clone() .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); + + // NOTE: visible_seqno is intentionally a separate default counter, not the custom + // generator. It serves as the MVCC visibility watermark, updated via publish()/set() + // from the write seqno values — it tracks "up to which seqno are writes visible", + // not "generate the next seqno". let visible_seqno: SharedSequenceNumberGenerator = Arc::new(SequenceNumberCounter::default()); From 45979fd6d983b98e7d20e19688d4ffc88c3f398d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 14 Mar 2026 18:56:45 +0200 Subject: [PATCH 04/14] docs(builder): show seqno_generator usage in doctest example Replace default-only example with one that demonstrates creating and passing a custom generator via the builder API. --- src/builder.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 0f661b1e..18dd0cc3 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -202,11 +202,16 @@ impl Builder { /// /// ``` /// use fjall::Database; + /// use lsm_tree::{SharedSequenceNumberGenerator, SequenceNumberCounter}; /// use std::sync::Arc; /// /// # let folder = tempfile::tempdir()?; - /// // Use default generator (SequenceNumberCounter) - /// let db = Database::builder(&folder).open()?; + /// let generator: SharedSequenceNumberGenerator = + /// Arc::new(SequenceNumberCounter::default()); + /// + /// let db = Database::builder(&folder) + /// .seqno_generator(generator) + /// .open()?; /// # Ok::<(), fjall::Error>(()) /// ``` #[must_use] From 9cdad86bb5a862b5868b46ac51fc1579ae3a45ed Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 14 Mar 2026 19:25:31 +0200 Subject: [PATCH 05/14] test(seqno): tighten recovery assertion to check state-advancing calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assert fetch_max/set calls instead of get calls — get() may be called only for debug logging without advancing generator state. --- tests/custom_seqno_generator.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/custom_seqno_generator.rs b/tests/custom_seqno_generator.rs index f82d1a3d..abfe6ca4 100644 --- a/tests/custom_seqno_generator.rs +++ b/tests/custom_seqno_generator.rs @@ -91,7 +91,7 @@ fn custom_generator_recovery() -> fjall::Result<()> { db.persist(fjall::PersistMode::SyncAll)?; } - // Recover with a new custom generator — recovery calls .set()/.fetch_max() + // Recover with a new custom generator — recovery should call .set()/.fetch_max() { let generator = Arc::new(TrackingGenerator::new()); let db = Database::builder(&folder) @@ -104,12 +104,12 @@ fn custom_generator_recovery() -> fjall::Result<()> { assert_eq!(b"value1", &*tree.get("key1")?.unwrap()); assert_eq!(b"value2", &*tree.get("key2")?.unwrap()); - // Recovery used the generator (fetch_max or get) + // Recovery advanced the generator state (via fetch_max or set) let fetch_max_calls = generator.fetch_max_count.load(Ordering::Relaxed); - let get_calls = generator.get_count.load(Ordering::Relaxed); + let set_calls = generator.set_count.load(Ordering::Relaxed); assert!( - fetch_max_calls > 0 || get_calls > 0, - "Recovery should use the custom generator", + fetch_max_calls > 0 || set_calls > 0, + "Recovery should advance the custom generator state via fetch_max()/set()", ); // Can write after recovery From 251a7d085eb7fa540864b57ea44b47bba7d03349 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 05:14:29 +0200 Subject: [PATCH 06/14] fix(deps): use fork lsm-tree with SequenceNumberGenerator trait - Switch lsm-tree from crates.io to structured-world/lsm-tree fork - Fork's main branch includes merged PR #10 with the trait Relates to #4 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 29e29c64..1871370e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ __internal_whitebox = [] [dependencies] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" -lsm-tree = { version = "~3.1.1", default-features = false, features = [] } +lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } log = "0.4.27" tempfile = "3.20.0" dashmap = "6.1.0" From c713a47ca88f002c63c5bdaa2530bf859b274160 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 05:20:38 +0200 Subject: [PATCH 07/14] refactor(db): extract seqno init helper, clean unused imports - Remove unused SequenceNumberGenerator imports across 6 files - Remove unused Arc import from recovery.rs - Extract make_seqno_generators() to deduplicate recover/create_new - Capture test baseline after keyspace creation for accurate assertion - Add doc comments for fetch_add semantics and API design choices Relates to #4 --- src/batch/mod.rs | 2 +- src/builder.rs | 5 ++++ src/db.rs | 48 ++++++++++++++++----------------- src/keyspace/mod.rs | 2 +- src/meta_keyspace.rs | 4 +-- src/recovery.rs | 4 +-- src/snapshot_tracker.rs | 2 +- tests/custom_seqno_generator.rs | 11 ++++++-- 8 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/batch/mod.rs b/src/batch/mod.rs index 21f2d107..14011e5a 100644 --- a/src/batch/mod.rs +++ b/src/batch/mod.rs @@ -6,7 +6,7 @@ pub mod item; use crate::{Database, Keyspace, PersistMode}; use item::Item; -use lsm_tree::{AbstractTree, SequenceNumberGenerator, UserKey, UserValue, ValueType}; +use lsm_tree::{AbstractTree, UserKey, UserValue, ValueType}; use std::collections::HashSet; /// An atomic write batch diff --git a/src/builder.rs b/src/builder.rs index 8bf41237..905ed4a0 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -191,6 +191,11 @@ impl Builder { /// Sets a custom sequence number generator for write operations. /// + /// Takes `SharedSequenceNumberGenerator` (`Arc`) + /// directly because the generator must be shared across all keyspaces and the + /// internal storage layer. Accepting a generic would add a type parameter to + /// `Builder` that propagates through `Config` and `Database`. + /// /// By default, the database uses [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter), /// a simple atomic counter. Use this to plug in a custom generator, /// e.g., a Hybrid Logical Clock (HLC) for distributed databases. diff --git a/src/db.rs b/src/db.rs index 74279e25..6535cb9b 100644 --- a/src/db.rs +++ b/src/db.rs @@ -23,9 +23,7 @@ use crate::{ write_buffer_manager::WriteBufferManager, HashMap, Keyspace, KeyspaceCreateOptions, }; -use lsm_tree::{ - AbstractTree, SequenceNumberCounter, SequenceNumberGenerator, SharedSequenceNumberGenerator, -}; +use lsm_tree::{AbstractTree, SequenceNumberCounter, SharedSequenceNumberGenerator}; use std::{ fs::remove_dir_all, path::Path, @@ -584,6 +582,26 @@ impl Database { self.supervisor.snapshot_tracker.get() } + /// Creates the write seqno generator and a separate MVCC visibility watermark. + /// + /// The write generator comes from the user-supplied config (or falls back to + /// the default [`SequenceNumberCounter`]). The visibility watermark is always + /// a fresh counter — it tracks "up to which seqno are writes visible", not + /// "generate the next seqno". + fn make_seqno_generators( + config: &Config, + ) -> (SharedSequenceNumberGenerator, SharedSequenceNumberGenerator) { + let seqno: SharedSequenceNumberGenerator = config + .seqno_generator + .clone() + .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); + + let visible_seqno: SharedSequenceNumberGenerator = + Arc::new(SequenceNumberCounter::default()); + + (seqno, visible_seqno) + } + fn check_version>(path: P) -> crate::Result<()> { let bytes = std::fs::read(path.as_ref().join(VERSION_MARKER))?; @@ -633,17 +651,7 @@ impl Database { let journal_manager = JournalManager::new(); - let seqno: SharedSequenceNumberGenerator = config - .seqno_generator - .clone() - .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); - - // NOTE: visible_seqno is intentionally a separate default counter, not the custom - // generator. It serves as the MVCC visibility watermark, updated via publish()/set() - // from the write seqno values — it tracks "up to which seqno are writes visible", - // not "generate the next seqno". - let visible_seqno: SharedSequenceNumberGenerator = - Arc::new(SequenceNumberCounter::default()); + let (seqno, visible_seqno) = Self::make_seqno_generators(&config); let keyspaces = Arc::new(RwLock::new(Keyspaces::with_capacity_and_hasher( 10, @@ -888,17 +896,7 @@ impl Database { fsync_directory(&keyspaces_folder_path)?; fsync_directory(&config.path)?; - let seqno: SharedSequenceNumberGenerator = config - .seqno_generator - .clone() - .unwrap_or_else(|| Arc::new(SequenceNumberCounter::default())); - - // NOTE: visible_seqno is intentionally a separate default counter, not the custom - // generator. It serves as the MVCC visibility watermark, updated via publish()/set() - // from the write seqno values — it tracks "up to which seqno are writes visible", - // not "generate the next seqno". - let visible_seqno: SharedSequenceNumberGenerator = - Arc::new(SequenceNumberCounter::default()); + let (seqno, visible_seqno) = Self::make_seqno_generators(&config); let keyspaces = Arc::new(RwLock::new(Keyspaces::with_capacity_and_hasher( 10, diff --git a/src/keyspace/mod.rs b/src/keyspace/mod.rs index 8b5a7649..83c894dd 100644 --- a/src/keyspace/mod.rs +++ b/src/keyspace/mod.rs @@ -20,7 +20,7 @@ use crate::{ worker_pool::WorkerMessage, Database, Guard, Iter, }; -use lsm_tree::{AbstractTree, AnyTree, SeqNo, SequenceNumberGenerator, UserKey, UserValue}; +use lsm_tree::{AbstractTree, AnyTree, SeqNo, UserKey, UserValue}; use options::CreateOptions; use std::{ ops::RangeBounds, diff --git a/src/meta_keyspace.rs b/src/meta_keyspace.rs index aeb74aed..721f308f 100644 --- a/src/meta_keyspace.rs +++ b/src/meta_keyspace.rs @@ -4,9 +4,7 @@ use crate::{db::Keyspaces, keyspace::InternalKeyspaceId, Keyspace}; use byteview::StrView; -use lsm_tree::{ - AbstractTree, AnyTree, SeqNo, SequenceNumberGenerator, SharedSequenceNumberGenerator, UserValue, -}; +use lsm_tree::{AbstractTree, AnyTree, SeqNo, SharedSequenceNumberGenerator, UserValue}; use std::sync::{Arc, RwLock, RwLockWriteGuard}; pub fn encode_config_key( diff --git a/src/recovery.rs b/src/recovery.rs index 5d40754b..527787aa 100644 --- a/src/recovery.rs +++ b/src/recovery.rs @@ -13,8 +13,8 @@ use crate::{ meta_keyspace::MetaKeyspace, Database, HashMap, Keyspace, }; -use lsm_tree::{AbstractTree, SequenceNumberGenerator}; -use std::{path::PathBuf, sync::Arc}; +use lsm_tree::AbstractTree; +use std::path::PathBuf; /// Recovers keyspaces pub fn recover_keyspaces(db: &Database, meta_keyspace: &MetaKeyspace) -> crate::Result<()> { diff --git a/src/snapshot_tracker.rs b/src/snapshot_tracker.rs index 8b23510a..ba25172f 100644 --- a/src/snapshot_tracker.rs +++ b/src/snapshot_tracker.rs @@ -4,7 +4,7 @@ use crate::{snapshot_nonce::SnapshotNonce, SeqNo}; use dashmap::DashMap; -use lsm_tree::{SequenceNumberGenerator, SharedSequenceNumberGenerator}; +use lsm_tree::SharedSequenceNumberGenerator; use std::sync::{atomic::AtomicU64, Arc, RwLock}; /// Keeps track of open snapshots diff --git a/tests/custom_seqno_generator.rs b/tests/custom_seqno_generator.rs index abfe6ca4..3b4e1c93 100644 --- a/tests/custom_seqno_generator.rs +++ b/tests/custom_seqno_generator.rs @@ -27,6 +27,8 @@ impl TrackingGenerator { } impl SequenceNumberGenerator for TrackingGenerator { + // NOTE: fetch_add returns the *previous* value — this matches SequenceNumberCounter's + // semantics where next() returns the current seqno then advances the counter. fn next(&self) -> SeqNo { self.next_count.fetch_add(1, Ordering::Relaxed); self.inner.fetch_add(1, Ordering::AcqRel) @@ -59,12 +61,17 @@ fn custom_generator_basic() -> fjall::Result<()> { let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + // Capture baseline after keyspace creation — metadata writes may have + // already advanced the generator before our inserts. + let baseline = generator.next_count.load(Ordering::Relaxed); + tree.insert("a", "hello")?; tree.insert("b", "world")?; + let after_inserts = generator.next_count.load(Ordering::Relaxed); assert!( - generator.next_count.load(Ordering::Relaxed) >= 2, - "Custom generator should be called for write operations", + after_inserts - baseline >= 2, + "Custom generator should be called at least twice for two inserts (baseline={baseline}, after={after_inserts})", ); assert_eq!(b"hello", &*tree.get("a")?.unwrap()); From 7d5372ec027e6599a7e54535a9307a2eaef820db Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 05:40:29 +0200 Subject: [PATCH 08/14] docs(builder): use fjall re-exports in doc examples and links - Doc example imports from fjall:: instead of lsm_tree:: - Doc links use intra-crate paths instead of lsm_tree:: prefix - Annotate fork git dependency in Cargo.toml Relates to #4 --- Cargo.toml | 2 ++ src/builder.rs | 4 ++-- src/db_config.rs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1871370e..2ebffe5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ __internal_whitebox = [] [dependencies] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" +# Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). +# Will switch back to crates.io once fjall-rs/lsm-tree#265 is released. lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } log = "0.4.27" tempfile = "3.20.0" diff --git a/src/builder.rs b/src/builder.rs index 905ed4a0..f960e290 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -196,7 +196,7 @@ impl Builder { /// internal storage layer. Accepting a generic would add a type parameter to /// `Builder` that propagates through `Config` and `Database`. /// - /// By default, the database uses [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter), + /// By default, the database uses [`SequenceNumberCounter`], /// a simple atomic counter. Use this to plug in a custom generator, /// e.g., a Hybrid Logical Clock (HLC) for distributed databases. /// @@ -208,7 +208,7 @@ impl Builder { /// /// ``` /// use fjall::Database; - /// use lsm_tree::{SharedSequenceNumberGenerator, SequenceNumberCounter}; + /// use fjall::{SharedSequenceNumberGenerator, SequenceNumberCounter}; /// use std::sync::Arc; /// /// # let folder = tempfile::tempdir()?; diff --git a/src/db_config.rs b/src/db_config.rs index 3a75f01c..1d55d9fb 100644 --- a/src/db_config.rs +++ b/src/db_config.rs @@ -54,7 +54,7 @@ pub struct Config { /// Custom sequence number generator. /// /// When set, this generator is used instead of the default - /// [`SequenceNumberCounter`](lsm_tree::SequenceNumberCounter). + /// [`SequenceNumberCounter`]. pub(crate) seqno_generator: Option, } From 9695eb5dd51d20a59abd1c39540f88607f84e530 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 05:49:56 +0200 Subject: [PATCH 09/14] chore(deps): annotate lsm-tree fork as alpha-phase dependency --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2ebffe5a..f89abdb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,8 @@ __internal_whitebox = [] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" # Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). -# Will switch back to crates.io once fjall-rs/lsm-tree#265 is released. +# Pinned to branch=main during alpha — will switch to crates.io version +# once fjall-rs/lsm-tree#265 is released. lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } log = "0.4.27" tempfile = "3.20.0" From 29fcab4709476e12a762390b92215a5c55fc399f Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 10:37:35 +0200 Subject: [PATCH 10/14] chore(deps): pin lsm-tree fork to specific rev --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f89abdb8..93c06adf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,9 @@ __internal_whitebox = [] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" # Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). -# Pinned to branch=main during alpha — will switch to crates.io version +# Pinned to a specific commit during alpha — will switch to crates.io version # once fjall-rs/lsm-tree#265 is released. -lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } +lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", rev = "25ffae70", default-features = false, features = [] } log = "0.4.27" tempfile = "3.20.0" dashmap = "6.1.0" From 08c3211dccc49df759eacf37efedd34aa77f9d87 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 10:39:46 +0200 Subject: [PATCH 11/14] revert: restore branch=main for lsm-tree fork dependency --- Cargo.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 93c06adf..8fd977ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,10 @@ __internal_whitebox = [] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" # Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). -# Pinned to a specific commit during alpha — will switch to crates.io version -# once fjall-rs/lsm-tree#265 is released. -lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", rev = "25ffae70", default-features = false, features = [] } +# Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). +# branch=main: active development, Dependabot tracks updates. +# Switch to crates.io once fjall-rs/lsm-tree#265 is released. +lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } log = "0.4.27" tempfile = "3.20.0" dashmap = "6.1.0" From 7e27512924019c7588bb9488e612f94af4ad94ef Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 10:40:47 +0200 Subject: [PATCH 12/14] docs(copilot): add dependency policy for lsm-tree branch=main --- .github/copilot-instructions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1f23f545..9437616f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -18,6 +18,16 @@ This is a **maintained fork** of [fjall-rs/fjall](https://github.com/fjall-rs/fj **Prefer issue suggestions over code suggestions for out-of-scope findings.** This keeps PRs focused and reviewable. +## Dependency Policy + +**`lsm-tree` uses `branch = "main"` intentionally.** This fork is in active alpha development alongside `structured-world/lsm-tree`. The `branch = "main"` git dependency is correct because: +- Both repos are under active co-development with frequent cross-repo changes +- Dependabot tracks and updates git branch dependencies +- Pinning to `rev` would block Dependabot and require manual bumps on every lsm-tree commit +- Will switch to a crates.io version once upstream fjall-rs/lsm-tree#265 is released + +**Do NOT suggest** pinning to `rev = "..."` or switching to a crates.io version for `lsm-tree`. This is a deliberate alpha-phase decision. + ## Rust Code Standards - **Unsafe code:** Prefer safe alternatives. `unsafe` requires `// SAFETY:` comment. From da039eb5014fdf80957b8b139e7d523f4976282c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 13:03:52 +0200 Subject: [PATCH 13/14] chore(deps): deduplicate lsm-tree annotation in Cargo.toml --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8fd977ba..90ff197d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ __internal_whitebox = [] byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = "~0.10.1" # Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). -# Fork with SequenceNumberGenerator trait (structured-world/lsm-tree#10). # branch=main: active development, Dependabot tracks updates. # Switch to crates.io once fjall-rs/lsm-tree#265 is released. lsm-tree = { git = "https://github.com/structured-world/lsm-tree.git", branch = "main", default-features = false, features = [] } From db0de327e3838639735dec0022d687e4f986a414 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Mon, 16 Mar 2026 13:43:35 +0200 Subject: [PATCH 14/14] docs(db): qualify SequenceNumberCounter intra-doc links --- src/builder.rs | 2 +- src/db_config.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index f960e290..9978a3f5 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -196,7 +196,7 @@ impl Builder { /// internal storage layer. Accepting a generic would add a type parameter to /// `Builder` that propagates through `Config` and `Database`. /// - /// By default, the database uses [`SequenceNumberCounter`], + /// By default, the database uses [`crate::SequenceNumberCounter`], /// a simple atomic counter. Use this to plug in a custom generator, /// e.g., a Hybrid Logical Clock (HLC) for distributed databases. /// diff --git a/src/db_config.rs b/src/db_config.rs index 1d55d9fb..b112ad41 100644 --- a/src/db_config.rs +++ b/src/db_config.rs @@ -54,7 +54,7 @@ pub struct Config { /// Custom sequence number generator. /// /// When set, this generator is used instead of the default - /// [`SequenceNumberCounter`]. + /// [`lsm_tree::SequenceNumberCounter`]. pub(crate) seqno_generator: Option, }