Skip to main content

absurder_sql/storage/
block_storage.rs

1#[allow(unused_imports)]
2use super::metadata::BlockMetadataPersist;
3use super::metadata::{ChecksumAlgorithm, ChecksumManager};
4#[cfg(any(
5    target_arch = "wasm32",
6    all(
7        not(target_arch = "wasm32"),
8        any(test, debug_assertions),
9        not(feature = "fs_persist")
10    )
11))]
12use super::vfs_sync;
13use crate::types::DatabaseError;
14#[cfg(target_arch = "wasm32")]
15use js_sys::Date;
16#[cfg(not(target_arch = "wasm32"))]
17use parking_lot::Mutex;
18#[cfg(target_arch = "wasm32")]
19use std::cell::RefCell;
20use std::collections::{HashMap, HashSet, VecDeque};
21#[allow(unused_imports)]
22use std::sync::{
23    Arc,
24    atomic::{AtomicBool, AtomicU64, Ordering},
25};
26use std::time::Duration;
27#[cfg(not(target_arch = "wasm32"))]
28use std::time::{Instant, SystemTime, UNIX_EPOCH};
29#[cfg(not(target_arch = "wasm32"))]
30use tokio::sync::mpsc;
31#[cfg(not(target_arch = "wasm32"))]
32use tokio::task::JoinHandle as TokioJoinHandle;
33
34// FS persistence imports (native only when feature is enabled)
35#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
36use std::path::PathBuf;
37
38// Global storage management moved to vfs_sync module
39
40// Persistent metadata storage moved to metadata module
41
42// Auto-sync messaging
43#[cfg(not(target_arch = "wasm32"))]
44#[derive(Debug)]
45pub(super) enum SyncRequest {
46    Timer(tokio::sync::oneshot::Sender<()>),
47    Debounce(tokio::sync::oneshot::Sender<()>),
48}
49
50#[derive(Clone, Debug, Default)]
51pub struct RecoveryOptions {
52    pub mode: RecoveryMode,
53    pub on_corruption: CorruptionAction,
54}
55
56#[derive(Clone, Debug, Default)]
57pub enum RecoveryMode {
58    #[default]
59    Full,
60    Sample {
61        count: usize,
62    },
63    Skip,
64}
65
66#[derive(Clone, Debug, Default)]
67pub enum CorruptionAction {
68    #[default]
69    Report,
70    Repair,
71    Fail,
72}
73
74#[derive(Clone, Debug, PartialEq)]
75pub enum CrashRecoveryAction {
76    NoActionNeeded,
77    Rollback,
78    Finalize,
79}
80
81#[derive(Clone, Debug, Default)]
82pub struct RecoveryReport {
83    pub total_blocks_verified: usize,
84    pub corrupted_blocks: Vec<u64>,
85    pub repaired_blocks: Vec<u64>,
86    pub verification_duration_ms: u64,
87}
88
89// On-disk JSON schema for fs_persist
90#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
91#[derive(serde::Serialize, serde::Deserialize, Default)]
92#[allow(dead_code)]
93struct FsMeta {
94    entries: Vec<(u64, BlockMetadataPersist)>,
95}
96
97#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
98#[derive(serde::Serialize, serde::Deserialize, Default)]
99#[allow(dead_code)]
100struct FsAlloc {
101    allocated: Vec<u64>,
102}
103
104#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
105#[derive(serde::Serialize, serde::Deserialize, Default)]
106#[allow(dead_code)]
107struct FsDealloc {
108    tombstones: Vec<u64>,
109}
110
111// Metadata mirror for native builds
112#[cfg(not(target_arch = "wasm32"))]
113thread_local! {
114    pub(super) static GLOBAL_METADATA_TEST: parking_lot::Mutex<HashMap<String, HashMap<u64, BlockMetadataPersist>>> = parking_lot::Mutex::new(HashMap::new());
115}
116
117// Commit marker mirror for native builds (when fs_persist is disabled)
118#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
119thread_local! {
120    static GLOBAL_COMMIT_MARKER_TEST: parking_lot::Mutex<HashMap<String, u64>> = parking_lot::Mutex::new(HashMap::new());
121}
122
123#[derive(Clone, Debug)]
124pub struct SyncPolicy {
125    pub interval_ms: Option<u64>,
126    pub max_dirty: Option<usize>,
127    pub max_dirty_bytes: Option<usize>,
128    pub debounce_ms: Option<u64>,
129    pub verify_after_write: bool,
130}
131
132#[cfg(not(target_arch = "wasm32"))]
133impl Drop for BlockStorage {
134    fn drop(&mut self) {
135        if let Some(stop) = &self.auto_sync_stop {
136            stop.store(true, Ordering::SeqCst);
137        }
138        if let Some(handle) = self.auto_sync_thread.take() {
139            let _ = handle.join();
140        }
141        if let Some(handle) = self.debounce_thread.take() {
142            let _ = handle.join();
143        }
144        if let Some(task) = self.tokio_timer_task.take() {
145            task.abort();
146        }
147        if let Some(task) = self.tokio_debounce_task.take() {
148            task.abort();
149        }
150        self.auto_sync_stop = None;
151    }
152}
153
154pub const BLOCK_SIZE: usize = 4096;
155#[allow(dead_code)]
156pub(super) const DEFAULT_CACHE_CAPACITY: usize = 128;
157#[allow(dead_code)]
158const STORE_NAME: &str = "sqlite_blocks";
159#[allow(dead_code)]
160const METADATA_STORE: &str = "metadata";
161
162// Helper macro to abstract lock API differences: RefCell for WASM, Mutex for native
163// WASM: Use try_borrow_mut and handle reentrancy at call sites
164#[cfg(target_arch = "wasm32")]
165macro_rules! lock_mutex {
166    ($mutex:expr) => {
167        $mutex
168            .try_borrow_mut()
169            .expect("RefCell borrow failed - reentrancy detected in block_storage.rs")
170    };
171}
172
173// Try-lock version for reentrancy-safe operations
174#[allow(unused_macros)]
175#[cfg(target_arch = "wasm32")]
176macro_rules! try_lock_mutex {
177    ($mutex:expr) => {
178        $mutex
179    };
180}
181
182#[cfg(not(target_arch = "wasm32"))]
183macro_rules! lock_mutex {
184    ($mutex:expr) => {
185        $mutex.lock()
186    };
187}
188
189// Safe read macro for WASM that handles reentrancy
190#[allow(unused_macros)]
191#[cfg(target_arch = "wasm32")]
192macro_rules! read_lock {
193    ($mutex:expr) => {
194        $mutex
195            .try_borrow()
196            .expect("RefCell borrow failed - likely reentrancy issue")
197    };
198}
199
200#[allow(unused_macros)]
201#[cfg(not(target_arch = "wasm32"))]
202macro_rules! read_lock {
203    ($mutex:expr) => {
204        $mutex.lock()
205    };
206}
207
208// Try-lock macro for reentrancy-safe operations
209#[allow(unused_macros)]
210#[cfg(target_arch = "wasm32")]
211macro_rules! try_lock_mutex {
212    ($mutex:expr) => {
213        $mutex.ok()
214    };
215}
216
217#[allow(unused_macros)]
218#[cfg(not(target_arch = "wasm32"))]
219macro_rules! try_lock_mutex {
220    ($mutex:expr) => {
221        Some($mutex.lock())
222    };
223}
224
225// Try-read macro for reentrancy-safe read operations
226#[allow(unused_macros)]
227#[cfg(target_arch = "wasm32")]
228macro_rules! try_read_lock {
229    ($mutex:expr) => {
230        $mutex.try_borrow().ok()
231    };
232}
233
234#[allow(unused_macros)]
235#[cfg(not(target_arch = "wasm32"))]
236macro_rules! try_read_lock {
237    ($mutex:expr) => {
238        Some($mutex.lock())
239    };
240}
241
242pub struct BlockStorage {
243    // WASM: RefCell for zero-cost interior mutability (single-threaded)
244    // Native: Mutex for thread safety
245    #[cfg(target_arch = "wasm32")]
246    pub(super) cache: RefCell<HashMap<u64, Vec<u8>>>,
247    #[cfg(not(target_arch = "wasm32"))]
248    pub(super) cache: Mutex<HashMap<u64, Vec<u8>>>,
249
250    #[cfg(target_arch = "wasm32")]
251    pub(super) dirty_blocks: Arc<RefCell<HashMap<u64, Vec<u8>>>>,
252    #[cfg(not(target_arch = "wasm32"))]
253    pub(super) dirty_blocks: Arc<Mutex<HashMap<u64, Vec<u8>>>>,
254
255    #[cfg(target_arch = "wasm32")]
256    pub(super) allocated_blocks: RefCell<HashSet<u64>>,
257    #[cfg(not(target_arch = "wasm32"))]
258    pub(super) allocated_blocks: Mutex<HashSet<u64>>,
259
260    #[cfg(target_arch = "wasm32")]
261    #[allow(dead_code)]
262    pub(super) deallocated_blocks: RefCell<HashSet<u64>>,
263    #[cfg(not(target_arch = "wasm32"))]
264    #[allow(dead_code)]
265    pub(super) deallocated_blocks: Mutex<HashSet<u64>>,
266    pub(super) next_block_id: AtomicU64,
267    pub(super) capacity: usize,
268
269    #[cfg(target_arch = "wasm32")]
270    pub(super) lru_order: RefCell<VecDeque<u64>>,
271    #[cfg(not(target_arch = "wasm32"))]
272    pub(super) lru_order: Mutex<VecDeque<u64>>,
273
274    // Checksum management (moved to metadata module)
275    pub(super) checksum_manager: ChecksumManager,
276    #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
277    pub(super) base_dir: PathBuf,
278    pub(super) db_name: String,
279
280    // Background sync settings
281    #[cfg(target_arch = "wasm32")]
282    pub(super) auto_sync_interval: RefCell<Option<Duration>>,
283    #[cfg(not(target_arch = "wasm32"))]
284    pub(super) auto_sync_interval: Mutex<Option<Duration>>,
285
286    #[cfg(not(target_arch = "wasm32"))]
287    pub(super) last_auto_sync: Instant,
288
289    #[cfg(target_arch = "wasm32")]
290    pub(super) policy: RefCell<Option<SyncPolicy>>,
291    #[cfg(not(target_arch = "wasm32"))]
292    pub(super) policy: Mutex<Option<SyncPolicy>>,
293    #[cfg(not(target_arch = "wasm32"))]
294    pub(super) auto_sync_stop: Option<Arc<AtomicBool>>,
295    #[cfg(not(target_arch = "wasm32"))]
296    pub(super) auto_sync_thread: Option<std::thread::JoinHandle<()>>,
297    #[cfg(not(target_arch = "wasm32"))]
298    pub(super) debounce_thread: Option<std::thread::JoinHandle<()>>,
299    #[cfg(not(target_arch = "wasm32"))]
300    pub(super) tokio_timer_task: Option<TokioJoinHandle<()>>,
301    #[cfg(not(target_arch = "wasm32"))]
302    pub(super) tokio_debounce_task: Option<TokioJoinHandle<()>>,
303    #[cfg(not(target_arch = "wasm32"))]
304    pub(super) last_write_ms: Arc<AtomicU64>,
305    #[cfg(not(target_arch = "wasm32"))]
306    pub(super) threshold_hit: Arc<AtomicBool>,
307    #[cfg(not(target_arch = "wasm32"))]
308    pub(super) sync_count: Arc<AtomicU64>,
309    #[cfg(not(target_arch = "wasm32"))]
310    pub(super) timer_sync_count: Arc<AtomicU64>,
311    #[cfg(not(target_arch = "wasm32"))]
312    pub(super) debounce_sync_count: Arc<AtomicU64>,
313    #[cfg(not(target_arch = "wasm32"))]
314    pub(super) last_sync_duration_ms: Arc<AtomicU64>,
315
316    // Auto-sync channel for real sync operations
317    #[cfg(not(target_arch = "wasm32"))]
318    pub(super) sync_sender: Option<mpsc::UnboundedSender<SyncRequest>>,
319    #[cfg(not(target_arch = "wasm32"))]
320    pub(super) sync_receiver: Option<mpsc::UnboundedReceiver<SyncRequest>>,
321
322    // Startup recovery report
323    pub(super) recovery_report: RecoveryReport,
324
325    // Leader election manager (WASM only) - wrapped in RefCell for interior mutability
326    #[cfg(target_arch = "wasm32")]
327    pub leader_election: std::cell::RefCell<Option<super::leader_election::LeaderElectionManager>>,
328
329    // Observability manager
330    pub(super) observability: super::observability::ObservabilityManager,
331
332    // Telemetry metrics (optional)
333    #[cfg(feature = "telemetry")]
334    pub(super) metrics: Option<crate::telemetry::Metrics>,
335}
336
337impl BlockStorage {
338    /// Create a new BlockStorage synchronously without IndexedDB restoration
339    /// Used for auto-registration in VFS when existing data is detected
340    #[cfg(target_arch = "wasm32")]
341    pub fn new_sync(db_name: &str) -> Self {
342        log::info!(
343            "Creating BlockStorage synchronously for database: {}",
344            db_name
345        );
346
347        // Load existing data from GLOBAL_STORAGE to support multi-connection scenarios
348        use crate::storage::vfs_sync::with_global_storage;
349        let (cache, allocated_blocks, max_block_id) = with_global_storage(|storage_map| {
350            if let Some(db_storage) = storage_map.borrow().get(db_name) {
351                let cache = db_storage.clone();
352                let allocated = db_storage.keys().copied().collect::<HashSet<_>>();
353                let max_id = db_storage.keys().max().copied().unwrap_or(0);
354                (cache, allocated, max_id)
355            } else {
356                (HashMap::new(), HashSet::new(), 0)
357            }
358        });
359
360        log::info!(
361            "Loaded {} blocks from GLOBAL_STORAGE for {} (max_block_id={})",
362            cache.len(),
363            db_name,
364            max_block_id
365        );
366
367        // CRITICAL: Always reload checksums from GLOBAL_METADATA since cache might be stale
368        // This handles the case where import writes new data after close() reloaded cache
369        use crate::storage::vfs_sync::with_global_metadata;
370        let checksum_manager = with_global_metadata(|metadata_map| {
371            if let Some(db_metadata) = metadata_map.borrow().get(db_name) {
372                let mut checksums = HashMap::new();
373                let mut algos = HashMap::new();
374                for (block_id, meta) in db_metadata {
375                    checksums.insert(*block_id, meta.checksum);
376                    algos.insert(*block_id, meta.algo);
377                }
378                ChecksumManager::with_data(checksums, algos, ChecksumAlgorithm::FastHash)
379            } else {
380                ChecksumManager::new(ChecksumAlgorithm::FastHash)
381            }
382        });
383
384        Self {
385            cache: RefCell::new(cache),
386            dirty_blocks: Arc::new(RefCell::new(HashMap::new())),
387            allocated_blocks: RefCell::new(allocated_blocks),
388            deallocated_blocks: RefCell::new(HashSet::new()),
389            next_block_id: AtomicU64::new(max_block_id + 1),
390            capacity: 128,
391            lru_order: RefCell::new(VecDeque::new()),
392            checksum_manager,
393            db_name: db_name.to_string(),
394            auto_sync_interval: RefCell::new(None),
395            policy: RefCell::new(None),
396            #[cfg(not(target_arch = "wasm32"))]
397            last_auto_sync: Instant::now(),
398            #[cfg(not(target_arch = "wasm32"))]
399            auto_sync_stop: None,
400            #[cfg(not(target_arch = "wasm32"))]
401            auto_sync_thread: None,
402            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
403            base_dir: std::path::PathBuf::from(
404                std::env::var("ABSURDERSQL_FS_BASE")
405                    .unwrap_or_else(|_| "./test_storage".to_string()),
406            ),
407            #[cfg(not(target_arch = "wasm32"))]
408            debounce_thread: None,
409            #[cfg(not(target_arch = "wasm32"))]
410            tokio_timer_task: None,
411            #[cfg(not(target_arch = "wasm32"))]
412            tokio_debounce_task: None,
413            #[cfg(not(target_arch = "wasm32"))]
414            last_write_ms: Arc::new(AtomicU64::new(0)),
415            #[cfg(not(target_arch = "wasm32"))]
416            threshold_hit: Arc::new(AtomicBool::new(false)),
417            #[cfg(not(target_arch = "wasm32"))]
418            sync_count: Arc::new(AtomicU64::new(0)),
419            #[cfg(not(target_arch = "wasm32"))]
420            timer_sync_count: Arc::new(AtomicU64::new(0)),
421            #[cfg(not(target_arch = "wasm32"))]
422            debounce_sync_count: Arc::new(AtomicU64::new(0)),
423            #[cfg(not(target_arch = "wasm32"))]
424            last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
425            #[cfg(not(target_arch = "wasm32"))]
426            sync_sender: None,
427            #[cfg(not(target_arch = "wasm32"))]
428            sync_receiver: None,
429            recovery_report: RecoveryReport::default(),
430            #[cfg(target_arch = "wasm32")]
431            leader_election: std::cell::RefCell::new(None),
432            observability: super::observability::ObservabilityManager::new(),
433            #[cfg(feature = "telemetry")]
434            metrics: None,
435        }
436    }
437
438    #[cfg(target_arch = "wasm32")]
439    pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
440        super::constructors::new_wasm(db_name).await
441    }
442
443    #[cfg(not(target_arch = "wasm32"))]
444    pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
445        // CRITICAL: Use centralized normalize_db_name to match VFS and Database.name
446        let db_name = crate::utils::normalize_db_name(db_name);
447        let db_name = db_name.as_str();
448        log::info!("Creating BlockStorage for database: {}", db_name);
449
450        // Initialize allocation tracking for native
451        let (allocated_blocks, next_block_id) = {
452            #[cfg(feature = "fs_persist")]
453            {
454                // fs_persist: restore allocation from filesystem
455                let mut allocated_blocks = HashSet::new();
456                let mut next_block_id: u64 = 1;
457
458                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
459                    .unwrap_or_else(|_| "./test_storage".to_string());
460                let mut alloc_path = std::path::PathBuf::from(base_path);
461                alloc_path.push(db_name);
462                alloc_path.push("allocations.json");
463
464                if let Ok(content) = std::fs::read_to_string(&alloc_path) {
465                    if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
466                        if let Some(allocated_array) = alloc_data["allocated"].as_array() {
467                            for block_id_val in allocated_array {
468                                if let Some(block_id) = block_id_val.as_u64() {
469                                    allocated_blocks.insert(block_id);
470                                }
471                            }
472                            next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
473                        }
474                    }
475                }
476
477                (allocated_blocks, next_block_id)
478            }
479
480            #[cfg(not(feature = "fs_persist"))]
481            {
482                // Native non-fs_persist: restore allocation from global storage
483                let mut allocated_blocks = HashSet::new();
484                let mut next_block_id: u64 = 1;
485
486                super::vfs_sync::with_global_allocation_map(|allocation_map| {
487                    if let Some(existing_allocations) = allocation_map.borrow().get(db_name) {
488                        allocated_blocks = existing_allocations.clone();
489                        next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
490                        log::info!(
491                            "Restored {} allocated blocks for database: {}",
492                            allocated_blocks.len(),
493                            db_name
494                        );
495                    }
496                });
497
498                (allocated_blocks, next_block_id)
499            }
500        };
501
502        // Initialize checksums and checksum algorithms
503        let checksums_init: HashMap<u64, u64> = {
504            #[cfg(feature = "fs_persist")]
505            {
506                let mut map = HashMap::new();
507                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
508                    .unwrap_or_else(|_| "./test_storage".to_string());
509                let mut meta_path = std::path::PathBuf::from(base_path);
510                meta_path.push(db_name);
511                meta_path.push("metadata.json");
512
513                if let Ok(content) = std::fs::read_to_string(&meta_path) {
514                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
515                        if let Some(entries) = meta_data["entries"].as_array() {
516                            for entry in entries {
517                                if let Some(arr) = entry.as_array() {
518                                    if let (Some(block_id), Some(obj)) = (
519                                        arr.first().and_then(|v| v.as_u64()),
520                                        arr.get(1).and_then(|v| v.as_object()),
521                                    ) {
522                                        if let Some(checksum) =
523                                            obj.get("checksum").and_then(|v| v.as_u64())
524                                        {
525                                            map.insert(block_id, checksum);
526                                        }
527                                    }
528                                }
529                            }
530                        }
531                    }
532                }
533                map
534            }
535
536            #[cfg(not(feature = "fs_persist"))]
537            {
538                // Native non-fs_persist: restore checksums from global test storage
539                let mut map = HashMap::new();
540                GLOBAL_METADATA_TEST.with(|meta| {
541                    let meta_map = meta.lock();
542                    if let Some(db_meta) = meta_map.get(db_name) {
543                        for (block_id, metadata) in db_meta.iter() {
544                            map.insert(*block_id, metadata.checksum);
545                        }
546                    }
547                });
548                map
549            }
550        };
551
552        let checksum_algos_init: HashMap<u64, ChecksumAlgorithm> = {
553            #[cfg(feature = "fs_persist")]
554            {
555                let mut map = HashMap::new();
556                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
557                    .unwrap_or_else(|_| "./test_storage".to_string());
558                let mut meta_path = std::path::PathBuf::from(base_path);
559                meta_path.push(db_name);
560                meta_path.push("metadata.json");
561
562                if let Ok(content) = std::fs::read_to_string(&meta_path) {
563                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
564                        if let Some(entries) = meta_data["entries"].as_array() {
565                            for entry in entries {
566                                if let Some(arr) = entry.as_array() {
567                                    if let (Some(block_id), Some(obj)) = (
568                                        arr.first().and_then(|v| v.as_u64()),
569                                        arr.get(1).and_then(|v| v.as_object()),
570                                    ) {
571                                        let algo = obj
572                                            .get("algo")
573                                            .and_then(|v| v.as_str())
574                                            .and_then(|s| match s {
575                                                "CRC32" => Some(ChecksumAlgorithm::CRC32),
576                                                "FastHash" => Some(ChecksumAlgorithm::FastHash),
577                                                _ => None,
578                                            })
579                                            .unwrap_or(ChecksumAlgorithm::FastHash);
580                                        map.insert(block_id, algo);
581                                    }
582                                }
583                            }
584                        }
585                    }
586                }
587                map
588            }
589
590            #[cfg(not(feature = "fs_persist"))]
591            {
592                // Native non-fs_persist: restore algorithms from global test storage
593                let mut map = HashMap::new();
594                GLOBAL_METADATA_TEST.with(|meta| {
595                    let meta_map = meta.lock();
596                    if let Some(db_meta) = meta_map.get(db_name) {
597                        for (block_id, metadata) in db_meta.iter() {
598                            map.insert(*block_id, metadata.algo);
599                        }
600                    }
601                });
602                map
603            }
604        };
605
606        // Determine default checksum algorithm from environment (fs_persist native), fallback to FastHash
607        #[cfg(feature = "fs_persist")]
608        let checksum_algo_default = match std::env::var("DATASYNC_CHECKSUM_ALGO").ok().as_deref() {
609            Some("CRC32") => ChecksumAlgorithm::CRC32,
610            _ => ChecksumAlgorithm::FastHash,
611        };
612        #[cfg(not(feature = "fs_persist"))]
613        let checksum_algo_default = ChecksumAlgorithm::FastHash;
614
615        // Load deallocated blocks from filesystem
616        let deallocated_blocks_init: HashSet<u64> = {
617            #[cfg(feature = "fs_persist")]
618            {
619                let mut set = HashSet::new();
620                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
621                    .unwrap_or_else(|_| "./test_storage".to_string());
622                let mut path = std::path::PathBuf::from(base_path);
623                path.push(db_name);
624                let mut dealloc_path = path.clone();
625                dealloc_path.push("deallocated.json");
626                if let Ok(content) = std::fs::read_to_string(&dealloc_path) {
627                    if let Ok(dealloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
628                        if let Some(tombstones_array) = dealloc_data["tombstones"].as_array() {
629                            for tombstone_val in tombstones_array {
630                                if let Some(block_id) = tombstone_val.as_u64() {
631                                    set.insert(block_id);
632                                }
633                            }
634                        }
635                    }
636                }
637                set
638            }
639            #[cfg(not(feature = "fs_persist"))]
640            {
641                HashSet::new()
642            }
643        };
644
645        Ok(BlockStorage {
646            db_name: db_name.to_string(),
647            cache: Mutex::new(HashMap::new()),
648            lru_order: Mutex::new(VecDeque::new()),
649            capacity: 1000,
650            checksum_manager: ChecksumManager::with_data(
651                checksums_init,
652                checksum_algos_init,
653                checksum_algo_default,
654            ),
655            dirty_blocks: Arc::new(Mutex::new(HashMap::new())),
656            allocated_blocks: Mutex::new(allocated_blocks),
657            next_block_id: AtomicU64::new(next_block_id),
658            deallocated_blocks: Mutex::new(deallocated_blocks_init),
659            policy: Mutex::new(None),
660            auto_sync_interval: Mutex::new(None),
661            #[cfg(not(target_arch = "wasm32"))]
662            last_auto_sync: Instant::now(),
663            #[cfg(not(target_arch = "wasm32"))]
664            auto_sync_stop: None,
665            #[cfg(not(target_arch = "wasm32"))]
666            auto_sync_thread: None,
667            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
668            base_dir: std::path::PathBuf::from(
669                std::env::var("ABSURDERSQL_FS_BASE")
670                    .unwrap_or_else(|_| "./test_storage".to_string()),
671            ),
672            #[cfg(not(target_arch = "wasm32"))]
673            debounce_thread: None,
674            #[cfg(not(target_arch = "wasm32"))]
675            tokio_timer_task: None,
676            #[cfg(not(target_arch = "wasm32"))]
677            tokio_debounce_task: None,
678            #[cfg(not(target_arch = "wasm32"))]
679            last_write_ms: Arc::new(AtomicU64::new(0)),
680            #[cfg(not(target_arch = "wasm32"))]
681            threshold_hit: Arc::new(AtomicBool::new(false)),
682            #[cfg(not(target_arch = "wasm32"))]
683            sync_count: Arc::new(AtomicU64::new(0)),
684            #[cfg(not(target_arch = "wasm32"))]
685            timer_sync_count: Arc::new(AtomicU64::new(0)),
686            #[cfg(not(target_arch = "wasm32"))]
687            debounce_sync_count: Arc::new(AtomicU64::new(0)),
688            #[cfg(not(target_arch = "wasm32"))]
689            last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
690            #[cfg(not(target_arch = "wasm32"))]
691            sync_sender: None,
692            #[cfg(not(target_arch = "wasm32"))]
693            sync_receiver: None,
694            recovery_report: RecoveryReport::default(),
695            #[cfg(target_arch = "wasm32")]
696            leader_election: std::cell::RefCell::new(None),
697            observability: super::observability::ObservabilityManager::new(),
698            #[cfg(feature = "telemetry")]
699            metrics: None,
700        })
701    }
702
703    pub async fn new_with_capacity(db_name: &str, capacity: usize) -> Result<Self, DatabaseError> {
704        let mut s = Self::new(db_name).await?;
705        s.capacity = capacity;
706        Ok(s)
707    }
708
709    pub async fn new_with_recovery_options(
710        db_name: &str,
711        recovery_opts: RecoveryOptions,
712    ) -> Result<Self, DatabaseError> {
713        let mut storage = Self::new(db_name).await?;
714
715        storage.perform_startup_recovery(recovery_opts).await?;
716
717        Ok(storage)
718    }
719
720    pub fn get_recovery_report(&self) -> &RecoveryReport {
721        &self.recovery_report
722    }
723
724    async fn perform_startup_recovery(
725        &mut self,
726        opts: RecoveryOptions,
727    ) -> Result<(), DatabaseError> {
728        super::recovery::perform_startup_recovery(self, opts).await
729    }
730
731    pub(super) async fn get_blocks_for_verification(
732        &self,
733        mode: &RecoveryMode,
734    ) -> Result<Vec<u64>, DatabaseError> {
735        let all_blocks: Vec<u64> = lock_mutex!(self.allocated_blocks).iter().copied().collect();
736
737        match mode {
738            RecoveryMode::Full => Ok(all_blocks),
739            RecoveryMode::Sample { count } => {
740                let sample_count = (*count).min(all_blocks.len());
741                let mut sampled = all_blocks;
742                sampled.sort_unstable(); // Deterministic sampling
743                sampled.truncate(sample_count);
744                Ok(sampled)
745            }
746            RecoveryMode::Skip => Ok(Vec::new()),
747        }
748    }
749
750    pub(super) async fn verify_block_integrity(
751        &mut self,
752        block_id: u64,
753    ) -> Result<bool, DatabaseError> {
754        // Read the block data
755        let data = match self.read_block_from_storage(block_id).await {
756            Ok(data) => data,
757            Err(_) => {
758                log::warn!(
759                    "Could not read block {} for integrity verification",
760                    block_id
761                );
762                return Ok(false);
763            }
764        };
765
766        // Verify against stored checksum
767        match self.verify_against_stored_checksum(block_id, &data) {
768            Ok(()) => Ok(true),
769            Err(e) => {
770                log::warn!(
771                    "Block {} failed checksum verification: {}",
772                    block_id,
773                    e.message
774                );
775                Ok(false)
776            }
777        }
778    }
779
780    async fn read_block_from_storage(&mut self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
781        // Try to read from filesystem first (fs_persist mode)
782        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
783        {
784            let mut blocks_dir = self.base_dir.clone();
785            blocks_dir.push(&self.db_name);
786            blocks_dir.push("blocks");
787            let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
788
789            if let Ok(data) = std::fs::read(&block_file) {
790                if data.len() == BLOCK_SIZE {
791                    return Ok(data);
792                }
793            }
794        }
795
796        // Fallback to test storage for native tests
797        #[cfg(all(
798            not(target_arch = "wasm32"),
799            any(test, debug_assertions),
800            not(feature = "fs_persist")
801        ))]
802        {
803            let mut found_data = None;
804            vfs_sync::with_global_storage(|storage| {
805                #[cfg(target_arch = "wasm32")]
806                let storage_map = storage;
807                #[cfg(not(target_arch = "wasm32"))]
808                let storage_map = storage.borrow();
809                if let Some(db_storage) = storage_map.get(&self.db_name) {
810                    if let Some(data) = db_storage.get(&block_id) {
811                        found_data = Some(data.clone());
812                    }
813                }
814            });
815            if let Some(data) = found_data {
816                return Ok(data);
817            }
818        }
819
820        // WASM global storage
821        #[cfg(target_arch = "wasm32")]
822        {
823            let mut found_data = None;
824            vfs_sync::with_global_storage(|storage_map| {
825                if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
826                    if let Some(data) = db_storage.get(&block_id) {
827                        found_data = Some(data.clone());
828                    }
829                }
830            });
831            if let Some(data) = found_data {
832                return Ok(data);
833            }
834        }
835
836        Err(DatabaseError::new(
837            "BLOCK_NOT_FOUND",
838            &format!("Block {} not found in storage", block_id),
839        ))
840    }
841
842    pub(super) async fn repair_corrupted_block(
843        &mut self,
844        block_id: u64,
845    ) -> Result<bool, DatabaseError> {
846        log::info!("Attempting to repair corrupted block {}", block_id);
847
848        // For now, repair by removing the corrupted block and clearing its metadata
849        // In a real implementation, this might involve restoring from backup or rebuilding
850
851        // Remove from cache
852        lock_mutex!(self.cache).remove(&block_id);
853
854        // Remove checksum metadata
855        self.checksum_manager.remove_checksum(block_id);
856
857        // Remove from filesystem if fs_persist is enabled
858        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
859        {
860            let mut blocks_dir = self.base_dir.clone();
861            blocks_dir.push(&self.db_name);
862            blocks_dir.push("blocks");
863            let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
864            let _ = std::fs::remove_file(&block_file);
865        }
866
867        // Remove from test storage
868        #[cfg(all(
869            not(target_arch = "wasm32"),
870            any(test, debug_assertions),
871            not(feature = "fs_persist")
872        ))]
873        {
874            vfs_sync::with_global_storage(|storage| {
875                let mut storage_map = storage.borrow_mut();
876                if let Some(db_storage) = storage_map.get_mut(&self.db_name) {
877                    db_storage.remove(&block_id);
878                }
879            });
880        }
881
882        // Remove from WASM storage
883        #[cfg(target_arch = "wasm32")]
884        {
885            vfs_sync::with_global_storage(|storage_map| {
886                if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
887                    db_storage.remove(&block_id);
888                }
889            });
890        }
891
892        log::info!(
893            "Corrupted block {} has been removed (repair completed)",
894            block_id
895        );
896        Ok(true)
897    }
898
899    pub(super) fn touch_lru(&self, block_id: u64) {
900        // Remove any existing occurrence
901        let mut lru = lock_mutex!(self.lru_order);
902        if let Some(pos) = lru.iter().position(|&id| id == block_id) {
903            lru.remove(pos);
904        }
905        // Push as most-recent
906        lru.push_back(block_id);
907    }
908
909    pub(super) fn evict_if_needed(&self) {
910        // Evict clean LRU blocks until within capacity. Never evict dirty blocks.
911        loop {
912            // Check capacity and find victim in a single critical section
913            let victim_opt = {
914                let cache_guard = lock_mutex!(self.cache);
915                if cache_guard.len() <= self.capacity {
916                    break; // Within capacity, done
917                }
918
919                // Find the least-recent block that is NOT dirty
920                let dirty_guard = lock_mutex!(self.dirty_blocks);
921                let mut lru_guard = lock_mutex!(self.lru_order);
922
923                let victim_pos = lru_guard
924                    .iter()
925                    .position(|id| !dirty_guard.contains_key(id));
926
927                victim_pos.map(|pos| lru_guard.remove(pos).expect("valid pos"))
928                // Guards are dropped here
929            };
930
931            match victim_opt {
932                Some(victim) => {
933                    // Remove from cache (new lock acquisition)
934                    lock_mutex!(self.cache).remove(&victim);
935                }
936                None => {
937                    // All blocks are dirty; cannot evict. Allow temporary overflow.
938                    break;
939                }
940            }
941        }
942    }
943
944    #[inline]
945    #[cfg(target_arch = "wasm32")]
946    pub fn now_millis() -> u64 {
947        // Date::now() returns milliseconds since UNIX epoch as f64
948        Date::now() as u64
949    }
950
951    #[inline]
952    #[cfg(not(target_arch = "wasm32"))]
953    pub fn now_millis() -> u64 {
954        let now = SystemTime::now()
955            .duration_since(UNIX_EPOCH)
956            .unwrap_or_else(|_| Duration::from_millis(0));
957        now.as_millis() as u64
958    }
959
960    pub(super) fn verify_against_stored_checksum(
961        &self,
962        block_id: u64,
963        data: &[u8],
964    ) -> Result<(), DatabaseError> {
965        self.checksum_manager.validate_checksum(block_id, data)
966    }
967
968    /// Synchronous block read for environments that require sync access (e.g., VFS callbacks)
969    pub fn read_block_sync(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
970        // Implementation moved to io_operations module
971        super::io_operations::read_block_sync_impl(self, block_id)
972    }
973
974    pub async fn read_block(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
975        // Delegate to synchronous implementation (immediately ready)
976        self.read_block_sync(block_id)
977    }
978
979    /// Synchronous block write for environments that require sync access (e.g., VFS callbacks)
980    #[cfg(target_arch = "wasm32")]
981    pub fn write_block_sync(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
982        super::io_operations::write_block_sync_impl(self, block_id, data)
983    }
984
985    #[cfg(not(target_arch = "wasm32"))]
986    pub fn write_block_sync(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
987        super::io_operations::write_block_sync_impl(self, block_id, data)
988    }
989
990    #[cfg(target_arch = "wasm32")]
991    pub async fn write_block(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
992        self.write_block_sync(block_id, data)
993    }
994
995    #[cfg(not(target_arch = "wasm32"))]
996    pub async fn write_block(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
997        self.write_block_sync(block_id, data)?;
998
999        // If threshold was hit and debounce is NOT enabled, perform inline sync
1000        if self.threshold_hit.load(std::sync::atomic::Ordering::SeqCst) {
1001            let has_debounce = lock_mutex!(self.policy)
1002                .as_ref()
1003                .and_then(|p| p.debounce_ms)
1004                .is_some();
1005            if !has_debounce {
1006                log::debug!("Threshold hit without debounce: performing inline sync");
1007                self.sync_implementation()?;
1008                self.threshold_hit
1009                    .store(false, std::sync::atomic::Ordering::SeqCst);
1010            }
1011        }
1012
1013        Ok(())
1014    }
1015
1016    /// Synchronous batch write of blocks
1017    #[cfg(target_arch = "wasm32")]
1018    pub fn write_blocks_sync(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1019        self.maybe_auto_sync();
1020        for (block_id, data) in items {
1021            self.write_block_sync(block_id, data)?;
1022        }
1023        Ok(())
1024    }
1025
1026    #[cfg(not(target_arch = "wasm32"))]
1027    pub fn write_blocks_sync(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1028        self.maybe_auto_sync();
1029        for (block_id, data) in items {
1030            self.write_block_sync(block_id, data)?;
1031        }
1032        Ok(())
1033    }
1034
1035    /// Async batch write wrapper
1036    #[cfg(target_arch = "wasm32")]
1037    pub async fn write_blocks(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1038        self.write_blocks_sync(items)
1039    }
1040
1041    #[cfg(not(target_arch = "wasm32"))]
1042    pub async fn write_blocks(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1043        self.write_blocks_sync(items)
1044    }
1045
1046    /// Synchronous batch read of blocks, preserving input order
1047    pub fn read_blocks_sync(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1048        self.maybe_auto_sync();
1049        let mut results = Vec::with_capacity(block_ids.len());
1050        for &id in block_ids {
1051            results.push(self.read_block_sync(id)?);
1052        }
1053        Ok(results)
1054    }
1055
1056    /// Async batch read wrapper
1057    pub async fn read_blocks(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1058        self.read_blocks_sync(block_ids)
1059    }
1060
1061    /// Get block checksum for verification
1062    pub fn get_block_checksum(&self, block_id: u64) -> Option<u32> {
1063        self.checksum_manager
1064            .get_checksum(block_id)
1065            .map(|checksum| checksum as u32)
1066    }
1067
1068    /// Get current commit marker for this database (WASM only, for testing)
1069    #[cfg(target_arch = "wasm32")]
1070    pub fn get_commit_marker(&self) -> u64 {
1071        vfs_sync::with_global_commit_marker(|cm| {
1072            cm.borrow().get(&self.db_name).copied().unwrap_or(0)
1073        })
1074    }
1075
1076    /// Check if this database has any blocks in storage (WASM only)
1077    #[cfg(target_arch = "wasm32")]
1078    pub fn has_any_blocks(&self) -> bool {
1079        vfs_sync::with_global_storage(|storage_map| {
1080            storage_map
1081                .borrow()
1082                .get(&self.db_name)
1083                .map_or(false, |blocks| !blocks.is_empty())
1084        })
1085    }
1086
1087    pub async fn verify_block_checksum(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1088        // If cached, verify directly against cached bytes
1089        if let Some(bytes) = lock_mutex!(self.cache).get(&block_id).cloned() {
1090            return self.verify_against_stored_checksum(block_id, &bytes);
1091        }
1092        // Otherwise, a read will populate cache and also verify
1093        let data = self.read_block_sync(block_id)?;
1094        self.verify_against_stored_checksum(block_id, &data)
1095    }
1096
1097    // Always available for testing (integration tests need this in release mode)
1098    #[allow(unused_mut)]
1099    pub fn get_block_metadata_for_testing(&mut self) -> HashMap<u64, (u64, u32, u64)> {
1100        // Returns map of block_id -> (checksum, version, last_modified_ms)
1101        #[cfg(target_arch = "wasm32")]
1102        {
1103            let mut out = HashMap::new();
1104            vfs_sync::with_global_metadata(|meta_map| {
1105                if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1106                    for (bid, m) in db_meta.iter() {
1107                        out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1108                    }
1109                }
1110            });
1111            out
1112        }
1113        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
1114        {
1115            use super::fs_persist::FsMeta;
1116            use std::io::Read;
1117            let mut out = HashMap::new();
1118            let base: PathBuf = self.base_dir.clone();
1119            let mut db_dir = base.clone();
1120            db_dir.push(&self.db_name);
1121            let mut meta_path = db_dir.clone();
1122            meta_path.push("metadata.json");
1123            if let Ok(mut f) = std::fs::File::open(&meta_path) {
1124                let mut s = String::new();
1125                if f.read_to_string(&mut s).is_ok() {
1126                    if let Ok(meta) = serde_json::from_str::<FsMeta>(&s) {
1127                        for (block_id, metadata) in meta.entries {
1128                            out.insert(
1129                                block_id,
1130                                (
1131                                    metadata.checksum,
1132                                    metadata.version,
1133                                    metadata.last_modified_ms,
1134                                ),
1135                            );
1136                        }
1137                    }
1138                }
1139            }
1140            out
1141        }
1142        #[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
1143        {
1144            let mut out = HashMap::new();
1145            GLOBAL_METADATA_TEST.with(|meta| {
1146                let meta_map = meta.lock();
1147                if let Some(db_meta) = meta_map.get(&self.db_name) {
1148                    for (bid, m) in db_meta.iter() {
1149                        out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1150                    }
1151                }
1152            });
1153            out
1154        }
1155    }
1156
1157    // Always available for testing (integration tests need this in release mode)
1158    pub fn set_block_checksum_for_testing(&mut self, block_id: u64, checksum: u64) {
1159        self.checksum_manager
1160            .set_checksum_for_testing(block_id, checksum);
1161    }
1162
1163    /// Getter for dirty_blocks for fs_persist and auto_sync modules
1164    #[cfg(not(target_arch = "wasm32"))]
1165    pub(super) fn get_dirty_blocks(&self) -> &Arc<Mutex<HashMap<u64, Vec<u8>>>> {
1166        &self.dirty_blocks
1167    }
1168
1169    #[cfg(target_arch = "wasm32")]
1170    #[allow(invalid_reference_casting)]
1171    pub async fn sync(&self) -> Result<(), DatabaseError> {
1172        // WASM: Use unsafe cast to get mutable access
1173        let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1174        let result = self_mut.sync_implementation();
1175        // Give time for the spawned IndexedDB operations to complete
1176        wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve(
1177            &wasm_bindgen::JsValue::UNDEFINED,
1178        ))
1179        .await
1180        .ok();
1181        result
1182    }
1183
1184    #[cfg(not(target_arch = "wasm32"))]
1185    pub async fn sync(&mut self) -> Result<(), DatabaseError> {
1186        self.sync_implementation()
1187    }
1188
1189    /// Synchronous version of sync() for immediate persistence
1190    #[cfg(target_arch = "wasm32")]
1191    #[allow(invalid_reference_casting)]
1192    pub fn sync_now(&self) -> Result<(), DatabaseError> {
1193        // WASM can use &self due to RefCell interior mutability
1194        // Cast self to mutable for the implementation
1195        let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1196        self_mut.sync_implementation()
1197    }
1198
1199    #[cfg(not(target_arch = "wasm32"))]
1200    pub fn sync_now(&mut self) -> Result<(), DatabaseError> {
1201        self.sync_implementation()
1202    }
1203
1204    /// Internal sync implementation shared by sync() and sync_now()
1205    fn sync_implementation(&mut self) -> Result<(), DatabaseError> {
1206        super::sync_operations::sync_implementation_impl(self)
1207    }
1208
1209    /// Sync blocks to global storage without advancing commit marker
1210    /// Used by VFS x_sync callback to persist blocks but maintain commit marker lag
1211    #[cfg(target_arch = "wasm32")]
1212    pub fn sync_blocks_only(&mut self) -> Result<(), DatabaseError> {
1213        super::wasm_vfs_sync::sync_blocks_only(self)
1214    }
1215
1216    /// Async version of sync for WASM that properly awaits IndexedDB persistence
1217    #[cfg(target_arch = "wasm32")]
1218    pub async fn sync_async(&self) -> Result<(), DatabaseError> {
1219        super::wasm_indexeddb::sync_async(self).await
1220    }
1221
1222    /// Drain all pending dirty blocks and stop background auto-sync (if enabled).
1223    /// Safe to call multiple times.
1224    #[cfg(target_arch = "wasm32")]
1225    #[allow(invalid_reference_casting)]
1226    pub fn drain_and_shutdown(&self) {
1227        // WASM: Use unsafe cast to get mutable access
1228        let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1229        if let Err(e) = self_mut.sync_now() {
1230            log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1231        }
1232        *lock_mutex!(self.auto_sync_interval) = None;
1233    }
1234
1235    #[cfg(not(target_arch = "wasm32"))]
1236    pub fn drain_and_shutdown(&mut self) {
1237        if let Err(e) = self.sync_now() {
1238            log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1239        }
1240        *lock_mutex!(self.auto_sync_interval) = None;
1241        #[cfg(not(target_arch = "wasm32"))]
1242        {
1243            if let Some(stop) = &self.auto_sync_stop {
1244                stop.store(true, Ordering::SeqCst);
1245            }
1246            if let Some(handle) = self.auto_sync_thread.take() {
1247                let _ = handle.join();
1248            }
1249            if let Some(handle) = self.debounce_thread.take() {
1250                let _ = handle.join();
1251            }
1252            if let Some(task) = self.tokio_timer_task.take() {
1253                task.abort();
1254            }
1255            if let Some(task) = self.tokio_debounce_task.take() {
1256                task.abort();
1257            }
1258            self.auto_sync_stop = None;
1259            self.threshold_hit.store(false, Ordering::SeqCst);
1260        }
1261    }
1262
1263    pub fn clear_cache(&self) {
1264        lock_mutex!(self.cache).clear();
1265        lock_mutex!(self.lru_order).clear();
1266    }
1267
1268    /// Handle notification that the database has been imported
1269    ///
1270    /// This method should be called after a database import to ensure
1271    /// that any cached data is invalidated and fresh data is read from storage.
1272    ///
1273    /// # Returns
1274    /// * `Ok(())` - Cache cleared successfully
1275    /// * `Err(DatabaseError)` - If cache clearing fails
1276    ///
1277    /// # Example
1278    /// ```rust,no_run
1279    /// # use absurder_sql::storage::block_storage::BlockStorage;
1280    /// # async fn example() -> Result<(), absurder_sql::types::DatabaseError> {
1281    /// let mut storage = BlockStorage::new("mydb").await?;
1282    /// // ... database is imported externally ...
1283    /// storage.on_database_import().await?;
1284    /// # Ok(())
1285    /// # }
1286    /// ```
1287    pub async fn on_database_import(&mut self) -> Result<(), DatabaseError> {
1288        log::info!(
1289            "Clearing cache for database '{}' after import",
1290            self.db_name
1291        );
1292
1293        // Clear the LRU cache to force re-reading from storage
1294        self.clear_cache();
1295
1296        // Also clear dirty blocks since they're now stale
1297        lock_mutex!(self.dirty_blocks).clear();
1298
1299        // Clear checksum manager's cache to reload from new metadata
1300        self.checksum_manager.clear_checksums();
1301
1302        // Reload allocated blocks from global storage/allocation map
1303        #[cfg(target_arch = "wasm32")]
1304        {
1305            use super::vfs_sync::with_global_allocation_map;
1306            *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1307                gam.borrow()
1308                    .get(&self.db_name)
1309                    .cloned()
1310                    .unwrap_or_else(std::collections::HashSet::new)
1311            });
1312            log::debug!(
1313                "Reloaded {} allocated blocks from global allocation map",
1314                lock_mutex!(self.allocated_blocks).len()
1315            );
1316
1317            // Checksums are now managed by ChecksumManager, which loads from metadata on demand
1318            log::debug!("Checksum data will be reloaded from metadata on next verification");
1319        }
1320
1321        #[cfg(not(target_arch = "wasm32"))]
1322        {
1323            #[cfg(feature = "fs_persist")]
1324            {
1325                // Reload from filesystem allocations.json
1326                let mut alloc_path = self.base_dir.clone();
1327                alloc_path.push(&self.db_name);
1328                alloc_path.push("allocations.json");
1329
1330                if let Ok(content) = std::fs::read_to_string(&alloc_path) {
1331                    if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
1332                        if let Some(allocated_array) = alloc_data["allocated"].as_array() {
1333                            lock_mutex!(self.allocated_blocks).clear();
1334                            for block_id_val in allocated_array {
1335                                if let Some(block_id) = block_id_val.as_u64() {
1336                                    lock_mutex!(self.allocated_blocks).insert(block_id);
1337                                }
1338                            }
1339                            log::debug!(
1340                                "Reloaded {} allocated blocks from filesystem",
1341                                lock_mutex!(self.allocated_blocks).len()
1342                            );
1343                        }
1344                    }
1345                }
1346
1347                // Reload checksums from filesystem metadata.json
1348                let mut meta_path = self.base_dir.clone();
1349                meta_path.push(&self.db_name);
1350                meta_path.push("metadata.json");
1351
1352                if let Ok(content) = std::fs::read_to_string(&meta_path) {
1353                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
1354                        if let Some(entries) = meta_data["entries"].as_array() {
1355                            let mut new_checksums = HashMap::new();
1356                            let mut new_algos = HashMap::new();
1357
1358                            for entry in entries {
1359                                if let (Some(block_id), Some(checksum), Some(algo_str)) = (
1360                                    entry[0].as_u64(),
1361                                    entry[1]["checksum"].as_u64(),
1362                                    entry[1]["algo"].as_str(),
1363                                ) {
1364                                    new_checksums.insert(block_id, checksum);
1365
1366                                    let algo = match algo_str {
1367                                        "CRC32" => super::metadata::ChecksumAlgorithm::CRC32,
1368                                        _ => super::metadata::ChecksumAlgorithm::FastHash,
1369                                    };
1370                                    new_algos.insert(block_id, algo);
1371                                }
1372                            }
1373
1374                            self.checksum_manager
1375                                .replace_all(new_checksums.clone(), new_algos);
1376                            log::debug!(
1377                                "Reloaded {} checksums from filesystem metadata",
1378                                new_checksums.len()
1379                            );
1380                        }
1381                    }
1382                } else {
1383                    log::debug!("No metadata file found, checksums will be empty after import");
1384                }
1385            }
1386
1387            #[cfg(not(feature = "fs_persist"))]
1388            {
1389                // Native test mode: reload from GLOBAL_ALLOCATION_MAP
1390                use super::vfs_sync::with_global_allocation_map;
1391
1392                *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1393                    #[cfg(target_arch = "wasm32")]
1394                    let map = gam;
1395                    #[cfg(not(target_arch = "wasm32"))]
1396                    let map = gam.borrow();
1397                    map.get(&self.db_name)
1398                        .cloned()
1399                        .unwrap_or_else(std::collections::HashSet::new)
1400                });
1401                log::debug!(
1402                    "Reloaded {} allocated blocks from global allocation map (native test)",
1403                    lock_mutex!(self.allocated_blocks).len()
1404                );
1405
1406                // Checksums are managed by ChecksumManager, loaded from metadata on demand
1407                log::debug!("Checksum data will be reloaded from metadata on next verification");
1408            }
1409        }
1410
1411        log::info!(
1412            "Cache and allocation state refreshed for '{}'",
1413            self.db_name
1414        );
1415
1416        Ok(())
1417    }
1418
1419    /// Reload cache from GLOBAL_STORAGE (WASM only, for multi-connection support)
1420    #[cfg(target_arch = "wasm32")]
1421    pub fn reload_cache_from_global_storage(&self) {
1422        use crate::storage::vfs_sync::{with_global_metadata, with_global_storage};
1423
1424        log::info!("[RELOAD] Starting cache reload for: {}", self.db_name);
1425
1426        #[cfg(target_arch = "wasm32")]
1427        web_sys::console::log_1(
1428            &format!("[RELOAD] BlockStorage.db_name = {}", self.db_name).into(),
1429        );
1430
1431        let fresh_cache = with_global_storage(|storage_map| {
1432            if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
1433                log::info!(
1434                    "[RELOAD] Found {} blocks in GLOBAL_STORAGE",
1435                    db_storage.len()
1436                );
1437                db_storage.clone()
1438            } else {
1439                log::warn!(
1440                    "[RELOAD] No blocks found in GLOBAL_STORAGE for {}",
1441                    self.db_name
1442                );
1443                std::collections::HashMap::new()
1444            }
1445        });
1446
1447        let block_count = fresh_cache.len();
1448        log::info!(
1449            "[RELOAD] Loading {} blocks into cache and marking as dirty",
1450            block_count
1451        );
1452
1453        // CRITICAL: Also reload checksums from GLOBAL_METADATA to match the fresh cache
1454        // Without this, cached reads will verify against stale checksums (e.g., after import)
1455        with_global_metadata(|metadata_map| {
1456            if let Some(db_metadata) = metadata_map.borrow().get(&self.db_name) {
1457                log::info!("[RELOAD] Found {} metadata entries", db_metadata.len());
1458                // Clear existing checksums
1459                self.checksum_manager.clear_checksums();
1460
1461                // Load checksums from metadata
1462                let mut new_checksums = std::collections::HashMap::new();
1463                let mut new_algos = std::collections::HashMap::new();
1464                for (block_id, meta) in db_metadata {
1465                    new_checksums.insert(*block_id, meta.checksum);
1466                    new_algos.insert(*block_id, meta.algo);
1467                }
1468                self.checksum_manager.replace_all(new_checksums, new_algos);
1469            } else {
1470                log::warn!("[RELOAD] No metadata found for {}", self.db_name);
1471                // No metadata exists, clear checksums
1472                self.checksum_manager.clear_checksums();
1473            }
1474        });
1475
1476        // Replace cache contents while preserving LRU order for blocks that still exist
1477        let old_lru = {
1478            let mut lru = lock_mutex!(self.lru_order);
1479            std::mem::replace(&mut *lru, std::collections::VecDeque::new())
1480        };
1481        lock_mutex!(self.cache).clear();
1482
1483        // Insert new cache data (no need to mark dirty - sync reads from GLOBAL_STORAGE)
1484        for (block_id, block_data) in fresh_cache {
1485            lock_mutex!(self.cache).insert(block_id, block_data);
1486        }
1487
1488        log::info!(
1489            "[RELOAD] Cache reloaded with {} blocks",
1490            lock_mutex!(self.cache).len()
1491        );
1492
1493        // Restore LRU order for blocks that still exist, then add new blocks
1494        for block_id in old_lru {
1495            if lock_mutex!(self.cache).contains_key(&block_id) {
1496                lock_mutex!(self.lru_order).push_back(block_id);
1497            }
1498        }
1499
1500        // Add any new blocks not in the old LRU order
1501        let block_ids: Vec<u64> = lock_mutex!(self.cache).keys().copied().collect();
1502        for block_id in block_ids {
1503            if !lock_mutex!(self.lru_order).contains(&block_id) {
1504                lock_mutex!(self.lru_order).push_back(block_id);
1505            }
1506        }
1507
1508        log::info!(
1509            "[RELOAD] Reload complete: {} blocks in cache, {} dirty",
1510            lock_mutex!(self.cache).len(),
1511            lock_mutex!(self.dirty_blocks).len()
1512        );
1513
1514        // CRITICAL DEBUG: Check if block 0 is valid SQLite header
1515        #[cfg(target_arch = "wasm32")]
1516        {
1517            if let Some(block_0) = lock_mutex!(self.cache).get(&0) {
1518                let is_valid = block_0.len() >= 16 && &block_0[0..16] == b"SQLite format 3\0";
1519                web_sys::console::log_1(
1520                    &format!(
1521                        "[RELOAD] Block 0 in cache: {} bytes, valid SQLite header: {}",
1522                        block_0.len(),
1523                        is_valid
1524                    )
1525                    .into(),
1526                );
1527                if block_0.len() >= 16 {
1528                    web_sys::console::log_1(
1529                        &format!("[RELOAD] Block 0 header [0..16]: {:02x?}", &block_0[0..16])
1530                            .into(),
1531                    );
1532                }
1533
1534                // CRITICAL: Check database size field at offset 24-27
1535                if block_0.len() >= 28 {
1536                    let db_size_bytes = [block_0[24], block_0[25], block_0[26], block_0[27]];
1537                    let db_size_pages = u32::from_be_bytes(db_size_bytes);
1538                    web_sys::console::log_1(
1539                        &format!(
1540                            "[RELOAD] Database size field (offset 24-27): {} pages ({:02x?})",
1541                            db_size_pages, db_size_bytes
1542                        )
1543                        .into(),
1544                    );
1545                }
1546
1547                // DUMP FULL HEADER for analysis
1548                if block_0.len() >= 100 {
1549                    web_sys::console::log_1(
1550                        &format!(
1551                            "[RELOAD] Full header dump [16-31]: {:02x?}",
1552                            &block_0[16..32]
1553                        )
1554                        .into(),
1555                    );
1556                    web_sys::console::log_1(
1557                        &format!(
1558                            "[RELOAD] Full header dump [32-47]: {:02x?}",
1559                            &block_0[32..48]
1560                        )
1561                        .into(),
1562                    );
1563                    web_sys::console::log_1(
1564                        &format!(
1565                            "[RELOAD] Full header dump [48-63]: {:02x?}",
1566                            &block_0[48..64]
1567                        )
1568                        .into(),
1569                    );
1570                    web_sys::console::log_1(
1571                        &format!(
1572                            "[RELOAD] Full header dump [64-79]: {:02x?}",
1573                            &block_0[64..80]
1574                        )
1575                        .into(),
1576                    );
1577                    web_sys::console::log_1(
1578                        &format!(
1579                            "[RELOAD] Full header dump [80-99]: {:02x?}",
1580                            &block_0[80..100]
1581                        )
1582                        .into(),
1583                    );
1584                }
1585            } else {
1586                web_sys::console::log_1(
1587                    &format!("[RELOAD] ERROR: Block 0 NOT in cache after reload!").into(),
1588                );
1589            }
1590        }
1591    }
1592
1593    pub fn get_cache_size(&self) -> usize {
1594        lock_mutex!(self.cache).len()
1595    }
1596
1597    pub fn get_dirty_count(&self) -> usize {
1598        lock_mutex!(self.dirty_blocks).len()
1599    }
1600
1601    pub fn get_db_name(&self) -> &str {
1602        &self.db_name
1603    }
1604
1605    pub fn is_cached(&self, block_id: u64) -> bool {
1606        lock_mutex!(self.cache).contains_key(&block_id)
1607    }
1608
1609    /// Allocate a new block and return its ID
1610    pub async fn allocate_block(&mut self) -> Result<u64, DatabaseError> {
1611        super::allocation::allocate_block_impl(self).await
1612    }
1613
1614    /// Deallocate a block and mark it as available for reuse
1615    pub async fn deallocate_block(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1616        super::allocation::deallocate_block_impl(self, block_id).await
1617    }
1618
1619    /// Get the number of currently allocated blocks
1620    pub fn get_allocated_count(&self) -> usize {
1621        lock_mutex!(self.allocated_blocks).len()
1622    }
1623
1624    /// Crash simulation: simulate crash during IndexedDB commit
1625    /// If `blocks_written` is true, blocks are written to IndexedDB but commit marker doesn't advance
1626    /// If `blocks_written` is false, crash occurs before blocks are written
1627    #[cfg(target_arch = "wasm32")]
1628    pub async fn crash_simulation_sync(
1629        &mut self,
1630        blocks_written: bool,
1631    ) -> Result<(), DatabaseError> {
1632        log::info!(
1633            "CRASH SIMULATION: Starting crash simulation with blocks_written={}",
1634            blocks_written
1635        );
1636
1637        if blocks_written {
1638            // Simulate crash after blocks are written but before commit marker advances
1639            // This is the most critical crash scenario to test
1640
1641            // Step 1: Write blocks to IndexedDB (simulate partial transaction completion)
1642            let dirty_blocks = {
1643                let dirty = lock_mutex!(self.dirty_blocks);
1644                dirty.clone()
1645            };
1646
1647            if !dirty_blocks.is_empty() {
1648                log::info!(
1649                    "CRASH SIMULATION: Writing {} blocks to IndexedDB before crash",
1650                    dirty_blocks.len()
1651                );
1652
1653                // Use the existing IndexedDB persistence logic but don't advance commit marker
1654                let metadata_to_persist: Vec<(u64, u64)> = dirty_blocks
1655                    .keys()
1656                    .map(|&block_id| {
1657                        let next_commit = self.get_commit_marker() + 1;
1658                        (block_id, next_commit)
1659                    })
1660                    .collect();
1661
1662                log::debug!(
1663                    "CRASH SIMULATION: About to call persist_to_indexeddb for {} blocks",
1664                    dirty_blocks.len()
1665                );
1666
1667                // Write blocks and metadata to IndexedDB
1668                super::wasm_indexeddb::persist_to_indexeddb(
1669                    &self.db_name,
1670                    dirty_blocks,
1671                    metadata_to_persist,
1672                )
1673                .await?;
1674
1675                log::info!("CRASH SIMULATION: persist_to_indexeddb completed successfully");
1676                log::info!(
1677                    "CRASH SIMULATION: Blocks written to IndexedDB, simulating crash before commit marker advance"
1678                );
1679
1680                // Clear dirty blocks (they're now in IndexedDB)
1681                lock_mutex!(self.dirty_blocks).clear();
1682
1683                // DON'T advance commit marker - this simulates the crash
1684                // In a real crash, the commit marker update would fail
1685
1686                return Ok(());
1687            } else {
1688                log::info!("CRASH SIMULATION: No dirty blocks to write");
1689                return Ok(());
1690            }
1691        } else {
1692            // Simulate crash before blocks are written
1693            log::info!("CRASH SIMULATION: Simulating crash before blocks are written to IndexedDB");
1694
1695            // Just return success - blocks remain dirty, nothing written to IndexedDB
1696            return Ok(());
1697        }
1698    }
1699
1700    /// Crash simulation: simulate partial block writes during IndexedDB commit
1701    /// Only specified blocks are written to IndexedDB before crash
1702    #[cfg(target_arch = "wasm32")]
1703    pub async fn crash_simulation_partial_sync(
1704        &mut self,
1705        blocks_to_write: &[u64],
1706    ) -> Result<(), DatabaseError> {
1707        log::info!(
1708            "CRASH SIMULATION: Starting partial crash simulation for {} blocks",
1709            blocks_to_write.len()
1710        );
1711
1712        let dirty_blocks = {
1713            let dirty = lock_mutex!(self.dirty_blocks);
1714            dirty.clone()
1715        };
1716
1717        // Filter to only the blocks we want to "successfully" write before crash
1718        let partial_blocks: std::collections::HashMap<u64, Vec<u8>> = dirty_blocks
1719            .into_iter()
1720            .filter(|(block_id, _)| blocks_to_write.contains(block_id))
1721            .collect();
1722
1723        if !partial_blocks.is_empty() {
1724            log::info!(
1725                "CRASH SIMULATION: Writing {} out of {} blocks before crash",
1726                partial_blocks.len(),
1727                blocks_to_write.len()
1728            );
1729
1730            let metadata_to_persist: Vec<(u64, u64)> = partial_blocks
1731                .keys()
1732                .map(|&block_id| {
1733                    let next_commit = self.get_commit_marker() + 1;
1734                    (block_id, next_commit)
1735                })
1736                .collect();
1737
1738            // Write only the partial blocks to IndexedDB
1739            super::wasm_indexeddb::persist_to_indexeddb(
1740                &self.db_name,
1741                partial_blocks.clone(),
1742                metadata_to_persist,
1743            )
1744            .await?;
1745
1746            // Remove only the written blocks from dirty_blocks
1747            {
1748                let mut dirty = lock_mutex!(self.dirty_blocks);
1749                for block_id in partial_blocks.keys() {
1750                    dirty.remove(block_id);
1751                }
1752            }
1753
1754            log::info!(
1755                "CRASH SIMULATION: Partial blocks written, simulating crash before commit marker advance"
1756            );
1757
1758            // DON'T advance commit marker - simulates crash during transaction
1759        }
1760
1761        Ok(())
1762    }
1763
1764    /// Perform crash recovery: detect and handle incomplete IndexedDB transactions
1765    /// This method detects inconsistencies between IndexedDB state and commit markers
1766    /// and either finalizes or rolls back incomplete transactions
1767    #[cfg(target_arch = "wasm32")]
1768    pub async fn perform_crash_recovery(&mut self) -> Result<CrashRecoveryAction, DatabaseError> {
1769        log::info!(
1770            "CRASH RECOVERY: Starting crash recovery scan for database: {}",
1771            self.db_name
1772        );
1773
1774        // Step 1: Get current commit marker
1775        let current_marker = self.get_commit_marker();
1776        log::info!("CRASH RECOVERY: Current commit marker: {}", current_marker);
1777
1778        // Step 2: Scan IndexedDB for blocks with versions > commit marker
1779        // These represent incomplete transactions that need recovery
1780        let inconsistent_blocks = self.scan_for_inconsistent_blocks(current_marker).await?;
1781
1782        if inconsistent_blocks.is_empty() {
1783            log::info!("CRASH RECOVERY: No inconsistent blocks found, system is consistent");
1784            return Ok(CrashRecoveryAction::NoActionNeeded);
1785        }
1786
1787        log::info!(
1788            "CRASH RECOVERY: Found {} inconsistent blocks that need recovery",
1789            inconsistent_blocks.len()
1790        );
1791
1792        // Step 3: Determine recovery action based on transaction completeness
1793        let recovery_action = self.determine_recovery_action(&inconsistent_blocks).await?;
1794
1795        match recovery_action {
1796            CrashRecoveryAction::Rollback => {
1797                log::info!("CRASH RECOVERY: Performing rollback of incomplete transaction");
1798                self.rollback_incomplete_transaction(&inconsistent_blocks)
1799                    .await?;
1800            }
1801            CrashRecoveryAction::Finalize => {
1802                log::info!("CRASH RECOVERY: Performing finalization of complete transaction");
1803                self.finalize_complete_transaction(&inconsistent_blocks)
1804                    .await?;
1805            }
1806            CrashRecoveryAction::NoActionNeeded => {
1807                // Already handled above
1808            }
1809        }
1810
1811        log::info!("CRASH RECOVERY: Recovery completed successfully");
1812        Ok(recovery_action)
1813    }
1814
1815    /// Scan IndexedDB for blocks with versions greater than the commit marker
1816    #[cfg(target_arch = "wasm32")]
1817    async fn scan_for_inconsistent_blocks(
1818        &self,
1819        commit_marker: u64,
1820    ) -> Result<Vec<(u64, u64)>, DatabaseError> {
1821        log::info!(
1822            "CRASH RECOVERY: Scanning for blocks with version > {}",
1823            commit_marker
1824        );
1825
1826        // This is a simplified implementation - in a real system we'd scan IndexedDB directly
1827        // For now, we'll check the global metadata storage
1828        let mut inconsistent_blocks = Vec::new();
1829
1830        vfs_sync::with_global_metadata(|meta_map| {
1831            if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1832                for (block_id, metadata) in db_meta.iter() {
1833                    if metadata.version as u64 > commit_marker {
1834                        log::info!(
1835                            "CRASH RECOVERY: Found inconsistent block {} with version {} > marker {}",
1836                            block_id,
1837                            metadata.version,
1838                            commit_marker
1839                        );
1840                        inconsistent_blocks.push((*block_id, metadata.version as u64));
1841                    }
1842                }
1843            }
1844        });
1845
1846        Ok(inconsistent_blocks)
1847    }
1848
1849    /// Determine whether to rollback or finalize based on transaction completeness
1850    #[cfg(target_arch = "wasm32")]
1851    async fn determine_recovery_action(
1852        &self,
1853        inconsistent_blocks: &[(u64, u64)],
1854    ) -> Result<CrashRecoveryAction, DatabaseError> {
1855        // Simple heuristic: if all inconsistent blocks have the same version (next expected commit),
1856        // then the transaction was likely complete and should be finalized.
1857        // Otherwise, rollback to maintain consistency.
1858
1859        let expected_next_commit = self.get_commit_marker() + 1;
1860        let all_same_version = inconsistent_blocks
1861            .iter()
1862            .all(|(_, version)| *version == expected_next_commit);
1863
1864        if all_same_version && !inconsistent_blocks.is_empty() {
1865            log::info!(
1866                "CRASH RECOVERY: All inconsistent blocks have expected version {}, finalizing transaction",
1867                expected_next_commit
1868            );
1869            Ok(CrashRecoveryAction::Finalize)
1870        } else {
1871            log::info!(
1872                "CRASH RECOVERY: Inconsistent block versions detected, rolling back transaction"
1873            );
1874            Ok(CrashRecoveryAction::Rollback)
1875        }
1876    }
1877
1878    /// Rollback incomplete transaction by removing inconsistent blocks
1879    #[cfg(target_arch = "wasm32")]
1880    async fn rollback_incomplete_transaction(
1881        &mut self,
1882        inconsistent_blocks: &[(u64, u64)],
1883    ) -> Result<(), DatabaseError> {
1884        log::info!(
1885            "CRASH RECOVERY: Rolling back {} inconsistent blocks",
1886            inconsistent_blocks.len()
1887        );
1888
1889        // Remove inconsistent blocks from global metadata
1890        vfs_sync::with_global_metadata(|meta_map| {
1891            if let Some(db_meta) = meta_map.borrow_mut().get_mut(&self.db_name) {
1892                for (block_id, _) in inconsistent_blocks {
1893                    log::info!(
1894                        "CRASH RECOVERY: Removing inconsistent block {} from metadata",
1895                        block_id
1896                    );
1897                    db_meta.remove(block_id);
1898                }
1899            }
1900        });
1901
1902        // Remove inconsistent blocks from global storage
1903        vfs_sync::with_global_storage(|storage_map| {
1904            if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
1905                for (block_id, _) in inconsistent_blocks {
1906                    log::info!(
1907                        "CRASH RECOVERY: Removing inconsistent block {} from global storage",
1908                        block_id
1909                    );
1910                    db_storage.remove(block_id);
1911                }
1912            }
1913        });
1914
1915        // Clear any cached data for these blocks
1916        for (block_id, _) in inconsistent_blocks {
1917            lock_mutex!(self.cache).remove(block_id);
1918            // Remove from LRU order
1919            lock_mutex!(self.lru_order).retain(|&id| id != *block_id);
1920        }
1921
1922        // Remove inconsistent blocks from IndexedDB to avoid accumulating orphaned data
1923        let block_ids_to_delete: Vec<u64> = inconsistent_blocks.iter().map(|(id, _)| *id).collect();
1924        if !block_ids_to_delete.is_empty() {
1925            log::info!(
1926                "CRASH RECOVERY: Deleting {} blocks from IndexedDB",
1927                block_ids_to_delete.len()
1928            );
1929            super::wasm_indexeddb::delete_blocks_from_indexeddb(
1930                &self.db_name,
1931                &block_ids_to_delete,
1932            )
1933            .await?;
1934            log::info!("CRASH RECOVERY: Successfully deleted blocks from IndexedDB");
1935        }
1936
1937        log::info!("CRASH RECOVERY: Rollback completed");
1938        Ok(())
1939    }
1940
1941    /// Finalize complete transaction by advancing commit marker
1942    #[cfg(target_arch = "wasm32")]
1943    async fn finalize_complete_transaction(
1944        &mut self,
1945        inconsistent_blocks: &[(u64, u64)],
1946    ) -> Result<(), DatabaseError> {
1947        log::info!(
1948            "CRASH RECOVERY: Finalizing transaction for {} blocks",
1949            inconsistent_blocks.len()
1950        );
1951
1952        // Find the target commit marker (should be consistent across all blocks)
1953        if let Some((_, target_version)) = inconsistent_blocks.first() {
1954            let new_commit_marker = *target_version;
1955
1956            // Advance the commit marker to make the blocks visible
1957            vfs_sync::with_global_commit_marker(|cm| {
1958                cm.borrow_mut()
1959                    .insert(self.db_name.clone(), new_commit_marker);
1960            });
1961
1962            log::info!(
1963                "CRASH RECOVERY: Advanced commit marker from {} to {}",
1964                self.get_commit_marker(),
1965                new_commit_marker
1966            );
1967
1968            // Update checksums for the finalized blocks
1969            for (block_id, _) in inconsistent_blocks {
1970                // Read the block data to compute and store checksum
1971                if let Ok(data) = self.read_block_sync(*block_id) {
1972                    self.checksum_manager.store_checksum(*block_id, &data);
1973                    log::info!(
1974                        "CRASH RECOVERY: Updated checksum for finalized block {}",
1975                        block_id
1976                    );
1977                }
1978            }
1979        }
1980
1981        log::info!("CRASH RECOVERY: Finalization completed");
1982        Ok(())
1983    }
1984
1985    // Leader Election Methods (WASM only)
1986
1987    /// Start leader election process
1988    #[cfg(target_arch = "wasm32")]
1989    pub async fn start_leader_election(&self) -> Result<(), DatabaseError> {
1990        if self.leader_election.borrow().is_none() {
1991            log::debug!(
1992                "BlockStorage::start_leader_election() - Creating new LeaderElectionManager for {}",
1993                self.db_name
1994            );
1995            let mut manager =
1996                super::leader_election::LeaderElectionManager::new(self.db_name.clone());
1997            log::debug!("BlockStorage::start_leader_election() - Calling manager.start_election()");
1998            manager.start_election().await?;
1999            log::debug!(
2000                "BlockStorage::start_leader_election() - Election started, storing manager"
2001            );
2002            *self.leader_election.borrow_mut() = Some(manager);
2003        } else {
2004            // If election is already running, force leadership takeover (requestLeadership)
2005            log::debug!(
2006                "BlockStorage::start_leader_election() - Election already exists, forcing leadership"
2007            );
2008            if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2009                manager.force_become_leader().await?;
2010            }
2011        }
2012        Ok(())
2013    }
2014
2015    /// Check if this instance WAS the leader WITHOUT triggering re-election
2016    /// Used during cleanup to avoid starting new elections
2017    #[cfg(target_arch = "wasm32")]
2018    pub fn was_leader(&self) -> bool {
2019        self.leader_election
2020            .borrow()
2021            .as_ref()
2022            .map(|m| m.state.borrow().is_leader)
2023            .unwrap_or(false)
2024    }
2025
2026    /// Stop heartbeat interval synchronously (for Drop implementation)
2027    /// This is idempotent and safe to call multiple times
2028    /// Silently skips if already stopped or if manager is borrowed
2029    #[cfg(target_arch = "wasm32")]
2030    pub fn stop_heartbeat_sync(&self) {
2031        // Use try_borrow_mut to avoid deadlock when multiple DBs drop simultaneously
2032        if let Ok(mut manager_ref) = self.leader_election.try_borrow_mut() {
2033            if let Some(ref mut manager) = *manager_ref {
2034                // Only clear if interval exists (idempotent)
2035                if let Some(interval_id) = manager.heartbeat_interval.take() {
2036                    if let Some(window) = web_sys::window() {
2037                        window.clear_interval_with_handle(interval_id);
2038                        web_sys::console::log_1(
2039                            &format!(
2040                                "[DROP] Cleared heartbeat interval {} for {}",
2041                                interval_id, self.db_name
2042                            )
2043                            .into(),
2044                        );
2045                    }
2046                }
2047                // Note: heartbeat closure is intentionally leaked (via forget())
2048                // and becomes a no-op when heartbeat_valid is set to false
2049            }
2050        } else {
2051            // Already borrowed (e.g., first DB is still dropping) - skip silently
2052            web_sys::console::log_1(
2053                &format!(
2054                    "[DROP] Skipping heartbeat stop for {} (already handled)",
2055                    self.db_name
2056                )
2057                .into(),
2058            );
2059        }
2060    }
2061
2062    /// Check if this instance is the leader (with re-election on lease expiry)
2063    #[cfg(target_arch = "wasm32")]
2064    pub async fn is_leader(&self) -> bool {
2065        // Start leader election if not already started
2066        if self.leader_election.borrow().is_none() {
2067            log::debug!(
2068                "BlockStorage::is_leader() - Starting leader election for {}",
2069                self.db_name
2070            );
2071            if let Err(e) = self.start_leader_election().await {
2072                log::error!("Failed to start leader election: {:?}", e);
2073                return false;
2074            }
2075            log::debug!("BlockStorage::is_leader() - Leader election started successfully");
2076        } else {
2077            log::debug!(
2078                "BlockStorage::is_leader() - Leader election already exists for {}",
2079                self.db_name
2080            );
2081        }
2082
2083        if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2084            let is_leader = manager.is_leader().await;
2085
2086            // If no current leader (lease expired), trigger re-election
2087            if !is_leader {
2088                let state = manager.state.borrow();
2089                if state.leader_id.is_none() {
2090                    log::debug!(
2091                        "No current leader for {} - triggering re-election",
2092                        self.db_name
2093                    );
2094                    drop(state);
2095                    let _ = manager.try_become_leader().await;
2096
2097                    // Start heartbeat if we became leader
2098                    let new_is_leader = manager.state.borrow().is_leader;
2099                    if new_is_leader && manager.heartbeat_interval.is_none() {
2100                        let _ = manager.start_heartbeat();
2101                    }
2102
2103                    log::debug!(
2104                        "is_leader() for {} = {} (after re-election)",
2105                        self.db_name,
2106                        new_is_leader
2107                    );
2108                    return new_is_leader;
2109                }
2110            }
2111
2112            log::debug!("is_leader() for {} = {}", self.db_name, is_leader);
2113            is_leader
2114        } else {
2115            log::debug!("No leader election manager for {}", self.db_name);
2116            false
2117        }
2118    }
2119
2120    /// Stop leader election (e.g., when tab is closing)
2121    #[cfg(target_arch = "wasm32")]
2122    pub async fn stop_leader_election(&self) -> Result<(), DatabaseError> {
2123        if let Some(mut manager) = self
2124            .leader_election
2125            .try_borrow_mut()
2126            .expect("Failed to borrow leader_election in stop_leader_election")
2127            .take()
2128        {
2129            manager.stop_election().await?;
2130        }
2131        Ok(())
2132    }
2133
2134    /// Send a leader heartbeat (for testing)
2135    #[cfg(target_arch = "wasm32")]
2136    pub async fn send_leader_heartbeat(&self) -> Result<(), DatabaseError> {
2137        if let Some(ref manager) = *self.leader_election.borrow() {
2138            manager.send_heartbeat().await
2139        } else {
2140            Err(DatabaseError::new(
2141                "LEADER_ELECTION_ERROR",
2142                "Leader election not started",
2143            ))
2144        }
2145    }
2146
2147    /// Get timestamp of last received leader heartbeat
2148    #[cfg(target_arch = "wasm32")]
2149    pub async fn get_last_leader_heartbeat(&self) -> Result<u64, DatabaseError> {
2150        if let Some(ref manager) = *self.leader_election.borrow() {
2151            Ok(manager.get_last_heartbeat().await)
2152        } else {
2153            Err(DatabaseError::new(
2154                "LEADER_ELECTION_ERROR",
2155                "Leader election not started",
2156            ))
2157        }
2158    }
2159
2160    // Observability Methods
2161
2162    /// Get comprehensive metrics for observability
2163    pub fn get_metrics(&self) -> super::observability::StorageMetrics {
2164        let dirty_count = self.get_dirty_count();
2165        let dirty_bytes = dirty_count * BLOCK_SIZE;
2166
2167        #[cfg(not(target_arch = "wasm32"))]
2168        let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2169            (
2170                self.sync_count.load(Ordering::SeqCst),
2171                self.timer_sync_count.load(Ordering::SeqCst),
2172                self.debounce_sync_count.load(Ordering::SeqCst),
2173                self.last_sync_duration_ms.load(Ordering::SeqCst),
2174            )
2175        };
2176
2177        #[cfg(target_arch = "wasm32")]
2178        let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2179            // For WASM, use observability manager for sync_count tracking
2180            (self.observability.get_sync_count(), 0, 0, 1)
2181        };
2182
2183        let error_count = self.observability.get_error_count();
2184        let checksum_failures = self.observability.get_checksum_failures();
2185
2186        // Calculate throughput and error rate
2187        let total_operations = sync_count + error_count;
2188        let (throughput_blocks_per_sec, throughput_bytes_per_sec) = self
2189            .observability
2190            .calculate_throughput(last_sync_duration_ms);
2191        let error_rate = self.observability.calculate_error_rate(total_operations);
2192
2193        super::observability::StorageMetrics {
2194            dirty_count,
2195            dirty_bytes,
2196            sync_count,
2197            timer_sync_count,
2198            debounce_sync_count,
2199            error_count,
2200            checksum_failures,
2201            last_sync_duration_ms,
2202            throughput_blocks_per_sec,
2203            throughput_bytes_per_sec,
2204            error_rate,
2205        }
2206    }
2207
2208    /// Set sync event callbacks
2209    #[cfg(not(target_arch = "wasm32"))]
2210    pub fn set_sync_callbacks(
2211        &mut self,
2212        on_sync_start: super::observability::SyncStartCallback,
2213        on_sync_success: super::observability::SyncSuccessCallback,
2214        on_sync_failure: super::observability::SyncFailureCallback,
2215    ) {
2216        self.observability.sync_start_callback = Some(on_sync_start);
2217        self.observability.sync_success_callback = Some(on_sync_success);
2218        self.observability.sync_failure_callback = Some(on_sync_failure);
2219    }
2220
2221    /// Set backpressure callback
2222    #[cfg(not(target_arch = "wasm32"))]
2223    pub fn set_backpressure_callback(
2224        &mut self,
2225        callback: super::observability::BackpressureCallback,
2226    ) {
2227        self.observability.backpressure_callback = Some(callback);
2228    }
2229
2230    /// Set error callback
2231    #[cfg(not(target_arch = "wasm32"))]
2232    pub fn set_error_callback(&mut self, callback: super::observability::ErrorCallback) {
2233        self.observability.error_callback = Some(callback);
2234    }
2235
2236    /// Set WASM sync success callback
2237    #[cfg(target_arch = "wasm32")]
2238    pub fn set_sync_success_callback(
2239        &mut self,
2240        callback: super::observability::WasmSyncSuccessCallback,
2241    ) {
2242        self.observability.wasm_sync_success_callback = Some(callback);
2243    }
2244
2245    /// Check if auto-sync is currently enabled
2246    pub fn is_auto_sync_enabled(&self) -> bool {
2247        lock_mutex!(self.auto_sync_interval).is_some()
2248    }
2249
2250    /// Get the current sync policy (if any)
2251    pub fn get_sync_policy(&self) -> Option<super::SyncPolicy> {
2252        lock_mutex!(self.policy).clone()
2253    }
2254
2255    /// Force synchronization with durability guarantees
2256    ///
2257    /// This method ensures that all dirty blocks are persisted to durable storage
2258    /// (IndexedDB in WASM, filesystem in native) and waits for the operation to complete.
2259    /// This is called by VFS xSync to provide SQLite's durability guarantees.
2260    pub async fn force_sync(&mut self) -> Result<(), DatabaseError> {
2261        log::info!("force_sync: Starting forced synchronization with durability guarantees");
2262
2263        let dirty_count = self.get_dirty_count();
2264        if dirty_count == 0 {
2265            log::debug!("force_sync: No dirty blocks to sync");
2266            return Ok(());
2267        }
2268
2269        log::info!(
2270            "force_sync: Syncing {} dirty blocks with durability guarantee",
2271            dirty_count
2272        );
2273
2274        // Just use the regular sync - it already waits for persistence in WASM
2275        self.sync().await?;
2276
2277        log::info!("force_sync: Successfully completed forced synchronization");
2278        Ok(())
2279    }
2280
2281    /// Set telemetry metrics (used for instrumentation)
2282    #[cfg(feature = "telemetry")]
2283    pub fn set_metrics(&mut self, metrics: Option<crate::telemetry::Metrics>) {
2284        self.metrics = metrics;
2285    }
2286
2287    /// Get telemetry metrics
2288    #[cfg(feature = "telemetry")]
2289    pub fn metrics(&self) -> Option<&crate::telemetry::Metrics> {
2290        self.metrics.as_ref()
2291    }
2292
2293    /// Create a test instance with minimal setup
2294    #[cfg(feature = "telemetry")]
2295    pub fn new_for_test() -> Self {
2296        Self {
2297            cache: Mutex::new(HashMap::new()),
2298            dirty_blocks: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2299            allocated_blocks: Mutex::new(HashSet::new()),
2300            deallocated_blocks: Mutex::new(HashSet::new()),
2301            next_block_id: AtomicU64::new(1),
2302            capacity: 128,
2303            lru_order: Mutex::new(VecDeque::new()),
2304            checksum_manager: crate::storage::metadata::ChecksumManager::new(
2305                crate::storage::metadata::ChecksumAlgorithm::FastHash,
2306            ),
2307            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
2308            base_dir: std::path::PathBuf::from("/tmp/test"),
2309            db_name: "test.db".to_string(),
2310            auto_sync_interval: Mutex::new(None),
2311            #[cfg(not(target_arch = "wasm32"))]
2312            last_auto_sync: std::time::Instant::now(),
2313            policy: Mutex::new(None),
2314            #[cfg(not(target_arch = "wasm32"))]
2315            auto_sync_stop: None,
2316            #[cfg(not(target_arch = "wasm32"))]
2317            auto_sync_thread: None,
2318            #[cfg(not(target_arch = "wasm32"))]
2319            debounce_thread: None,
2320            #[cfg(not(target_arch = "wasm32"))]
2321            tokio_timer_task: None,
2322            #[cfg(not(target_arch = "wasm32"))]
2323            tokio_debounce_task: None,
2324            #[cfg(not(target_arch = "wasm32"))]
2325            last_write_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2326            #[cfg(not(target_arch = "wasm32"))]
2327            threshold_hit: Arc::new(std::sync::atomic::AtomicBool::new(false)),
2328            #[cfg(not(target_arch = "wasm32"))]
2329            sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2330            #[cfg(not(target_arch = "wasm32"))]
2331            timer_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2332            #[cfg(not(target_arch = "wasm32"))]
2333            debounce_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2334            #[cfg(not(target_arch = "wasm32"))]
2335            last_sync_duration_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2336            #[cfg(not(target_arch = "wasm32"))]
2337            sync_sender: None,
2338            #[cfg(not(target_arch = "wasm32"))]
2339            sync_receiver: None,
2340            recovery_report: RecoveryReport::default(),
2341            #[cfg(target_arch = "wasm32")]
2342            leader_election: std::cell::RefCell::new(None),
2343            observability: super::observability::ObservabilityManager::new(),
2344            metrics: None,
2345        }
2346    }
2347}
2348
2349#[cfg(all(test, target_arch = "wasm32"))]
2350mod wasm_commit_marker_tests {
2351    use super::*;
2352    use wasm_bindgen_test::*;
2353
2354    wasm_bindgen_test_configure!(run_in_browser);
2355
2356    // Helper: set commit marker for a db name in WASM global
2357    fn set_commit_marker(db: &str, v: u64) {
2358        // Normalize name to match BlockStorage.db_name
2359        let db = crate::utils::normalize_db_name(db);
2360        super::vfs_sync::with_global_commit_marker(|cm| {
2361            cm.borrow_mut().insert(db, v);
2362        });
2363    }
2364
2365    // Helper: get commit marker for a db name in WASM global
2366    fn get_commit_marker(db: &str) -> u64 {
2367        // Normalize name to match BlockStorage.db_name
2368        let db = crate::utils::normalize_db_name(db);
2369        vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2370    }
2371
2372    #[wasm_bindgen_test]
2373    async fn gating_returns_zeroed_until_marker_catches_up_wasm() {
2374        let db = "cm_gating_wasm";
2375        let mut s = BlockStorage::new(db).await.expect("create storage");
2376
2377        // Write a block (starts at version 1, uncommitted)
2378        let bid = s.allocate_block().await.expect("alloc block");
2379        let data_v1 = vec![0x33u8; BLOCK_SIZE];
2380        s.write_block(bid, data_v1.clone()).await.expect("write v1");
2381
2382        // Before sync, commit marker is 0, block version is 1, so should be invisible
2383        s.clear_cache();
2384        let out0 = s.read_block(bid).await.expect("read before commit");
2385        assert_eq!(
2386            out0,
2387            vec![0u8; BLOCK_SIZE],
2388            "uncommitted data must read as zeroed"
2389        );
2390
2391        // After sync, commit marker advances to 1, block version is 1, so should be visible
2392        s.sync().await.expect("sync v1");
2393        s.clear_cache();
2394        let out1 = s.read_block(bid).await.expect("read after commit");
2395        assert_eq!(out1, data_v1, "committed data should be visible");
2396    }
2397
2398    #[wasm_bindgen_test]
2399    async fn invisible_blocks_skip_checksum_verification_wasm() {
2400        let db = "cm_checksum_skip_wasm";
2401        let mut s = BlockStorage::new(db).await.expect("create storage");
2402
2403        let bid = s.allocate_block().await.expect("alloc block");
2404        let data = vec![0x44u8; BLOCK_SIZE];
2405        s.write_block(bid, data).await.expect("write v1");
2406        s.sync().await.expect("sync v1"); // commit marker advances to 1, block version is 1
2407
2408        // Make the block invisible by moving commit marker back to 0
2409        set_commit_marker(db, 0);
2410
2411        // Corrupt the stored checksum; invisible reads must NOT verify checksum
2412        s.set_block_checksum_for_testing(bid, 1234567);
2413        s.clear_cache();
2414        let out = s
2415            .read_block(bid)
2416            .await
2417            .expect("read while invisible should not error");
2418        assert_eq!(
2419            out,
2420            vec![0u8; BLOCK_SIZE],
2421            "invisible block reads as zeroed"
2422        );
2423
2424        // Now make it visible again; checksum verification should trigger and fail
2425        set_commit_marker(db, 1);
2426        s.clear_cache();
2427        let err = s
2428            .read_block(bid)
2429            .await
2430            .expect_err("expected checksum mismatch once visible");
2431        assert_eq!(err.code, "CHECKSUM_MISMATCH");
2432    }
2433
2434    #[wasm_bindgen_test]
2435    async fn commit_marker_advances_and_versions_track_syncs_wasm() {
2436        let db = "cm_versions_wasm";
2437        let mut s = BlockStorage::new_with_capacity(db, 8)
2438            .await
2439            .expect("create storage");
2440
2441        let b1 = s.allocate_block().await.expect("alloc b1");
2442        let b2 = s.allocate_block().await.expect("alloc b2");
2443
2444        s.write_block(b1, vec![1u8; BLOCK_SIZE])
2445            .await
2446            .expect("write b1 v1");
2447        s.write_block(b2, vec![2u8; BLOCK_SIZE])
2448            .await
2449            .expect("write b2 v1");
2450        s.sync().await.expect("sync #1");
2451
2452        let cm1 = get_commit_marker(db);
2453        assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2454        let meta1 = s.get_block_metadata_for_testing();
2455        assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2456        assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2457
2458        // Update only b1 and sync again; only b1's version should bump
2459        s.write_block(b1, vec![3u8; BLOCK_SIZE])
2460            .await
2461            .expect("write b1 v2");
2462        s.sync().await.expect("sync #2");
2463
2464        let cm2 = get_commit_marker(db);
2465        assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2466        let meta2 = s.get_block_metadata_for_testing();
2467        assert_eq!(
2468            meta2.get(&b1).unwrap().1 as u64,
2469            cm2,
2470            "updated block tracks new version"
2471        );
2472        assert_eq!(
2473            meta2.get(&b2).unwrap().1 as u64,
2474            1,
2475            "unchanged block retains prior version"
2476        );
2477    }
2478}
2479
2480#[cfg(all(test, not(target_arch = "wasm32"), not(feature = "fs_persist")))]
2481mod commit_marker_tests {
2482    use super::*;
2483
2484    // Helper: set commit marker for a db name in test-global mirror
2485    fn set_commit_marker(db: &str, v: u64) {
2486        // Normalize name to match BlockStorage.db_name
2487        let db = crate::utils::normalize_db_name(db);
2488        super::vfs_sync::with_global_commit_marker(|cm| {
2489            cm.borrow_mut().insert(db, v);
2490        });
2491    }
2492
2493    // Helper: get commit marker for a db name in test-global mirror
2494    fn get_commit_marker(db: &str) -> u64 {
2495        // Normalize name to match BlockStorage.db_name
2496        let db = crate::utils::normalize_db_name(db);
2497        super::vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2498    }
2499
2500    #[tokio::test(flavor = "current_thread")]
2501    async fn gating_returns_zeroed_until_marker_catches_up() {
2502        let db = "cm_gating_basic";
2503        println!("DEBUG: Creating BlockStorage for {}", db);
2504        let mut s = BlockStorage::new(db).await.expect("create storage");
2505        println!("DEBUG: BlockStorage created successfully");
2506
2507        // Write a block (starts at version 1, uncommitted)
2508        let bid = s.allocate_block().await.expect("alloc block");
2509        println!("DEBUG: Allocated block {}", bid);
2510        let data_v1 = vec![0x11u8; BLOCK_SIZE];
2511        s.write_block(bid, data_v1.clone()).await.expect("write v1");
2512        println!("DEBUG: Wrote block {} with data", bid);
2513
2514        // Before sync, commit marker is 0, block version is 1, so should be invisible
2515        s.clear_cache();
2516        let out0 = s.read_block(bid).await.expect("read before commit");
2517        assert_eq!(
2518            out0,
2519            vec![0u8; BLOCK_SIZE],
2520            "uncommitted data must read as zeroed"
2521        );
2522        println!("DEBUG: Pre-sync read returned zeroed data as expected");
2523
2524        // After sync, commit marker advances to 1, block version is 1, so should be visible
2525        println!("DEBUG: About to call sync");
2526        s.sync().await.expect("sync v1");
2527        println!("DEBUG: Sync completed successfully");
2528
2529        // Debug: Check commit marker and metadata after sync
2530        let commit_marker = get_commit_marker(db);
2531        println!("DEBUG: Commit marker after sync: {}", commit_marker);
2532
2533        s.clear_cache();
2534        let out1 = s.read_block(bid).await.expect("read after commit");
2535
2536        // Debug: Print what we got vs what we expected
2537        println!("DEBUG: Expected data: {:?}", &data_v1[..8]);
2538        println!("DEBUG: Actual data: {:?}", &out1[..8]);
2539        println!(
2540            "DEBUG: Data lengths - expected: {}, actual: {}",
2541            data_v1.len(),
2542            out1.len()
2543        );
2544
2545        // Check if data matches without panicking
2546        let data_matches = out1 == data_v1;
2547        println!("DEBUG: Data matches: {}", data_matches);
2548
2549        if !data_matches {
2550            println!("DEBUG: Data mismatch detected - investigating further");
2551            // Check if it's all zeros (uncommitted)
2552            let is_all_zeros = out1.iter().all(|&b| b == 0);
2553            println!("DEBUG: Is all zeros: {}", is_all_zeros);
2554
2555            // Check metadata and commit marker state
2556            println!("DEBUG: Final commit marker: {}", get_commit_marker(db));
2557
2558            panic!("Data mismatch: expected committed data to be visible after sync");
2559        }
2560
2561        println!("DEBUG: Test passed - data is visible after commit");
2562    }
2563
2564    #[tokio::test(flavor = "current_thread")]
2565    async fn invisible_blocks_skip_checksum_verification() {
2566        let db = "cm_checksum_skip";
2567        let mut s = BlockStorage::new(db).await.expect("create storage");
2568
2569        let bid = s.allocate_block().await.expect("alloc block");
2570        let data = vec![0xAAu8; BLOCK_SIZE];
2571        s.write_block(bid, data.clone()).await.expect("write v1");
2572        s.sync().await.expect("sync v1"); // commit marker advances to 1, block version is 1
2573
2574        // Make the block invisible by moving commit marker back to 0
2575        set_commit_marker(db, 0);
2576
2577        // Corrupt the stored checksum; invisible reads must NOT verify checksum
2578        s.set_block_checksum_for_testing(bid, 1234567);
2579        s.clear_cache();
2580        let out = s
2581            .read_block(bid)
2582            .await
2583            .expect("read while invisible should not error");
2584        assert_eq!(
2585            out,
2586            vec![0u8; BLOCK_SIZE],
2587            "invisible block reads as zeroed"
2588        );
2589
2590        // Now make it visible again; checksum verification should trigger and fail
2591        set_commit_marker(db, 1);
2592        s.clear_cache();
2593        let err = s
2594            .read_block(bid)
2595            .await
2596            .expect_err("expected checksum mismatch once visible");
2597        assert_eq!(err.code, "CHECKSUM_MISMATCH");
2598    }
2599
2600    #[tokio::test(flavor = "current_thread")]
2601    async fn commit_marker_advances_and_versions_track_syncs() {
2602        let db = "cm_versions";
2603        let mut s = BlockStorage::new_with_capacity(db, 8)
2604            .await
2605            .expect("create storage");
2606
2607        let b1 = s.allocate_block().await.expect("alloc b1");
2608        let b2 = s.allocate_block().await.expect("alloc b2");
2609
2610        s.write_block(b1, vec![1u8; BLOCK_SIZE])
2611            .await
2612            .expect("write b1 v1");
2613        s.write_block(b2, vec![2u8; BLOCK_SIZE])
2614            .await
2615            .expect("write b2 v1");
2616        s.sync().await.expect("sync #1");
2617
2618        let cm1 = get_commit_marker(db);
2619        assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2620        let meta1 = s.get_block_metadata_for_testing();
2621        assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2622        assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2623
2624        // Update only b1 and sync again; only b1's version should bump
2625        s.write_block(b1, vec![3u8; BLOCK_SIZE])
2626            .await
2627            .expect("write b1 v2");
2628        s.sync().await.expect("sync #2");
2629
2630        let cm2 = get_commit_marker(db);
2631        assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2632        let meta2 = s.get_block_metadata_for_testing();
2633        assert_eq!(
2634            meta2.get(&b1).unwrap().1 as u64,
2635            cm2,
2636            "updated block tracks new version"
2637        );
2638        assert_eq!(
2639            meta2.get(&b2).unwrap().1 as u64,
2640            1,
2641            "unchanged block retains prior version"
2642        );
2643    }
2644}