Skip to main content

absurder_sql/storage/
block_storage.rs

1#[allow(unused_imports)]
2use super::metadata::BlockMetadataPersist;
3use super::metadata::{ChecksumAlgorithm, ChecksumManager};
4#[cfg(any(
5    target_arch = "wasm32",
6    all(
7        not(target_arch = "wasm32"),
8        any(test, debug_assertions),
9        not(feature = "fs_persist")
10    )
11))]
12use super::vfs_sync;
13use crate::types::DatabaseError;
14#[cfg(target_arch = "wasm32")]
15use js_sys::Date;
16#[cfg(not(target_arch = "wasm32"))]
17use parking_lot::Mutex;
18#[cfg(target_arch = "wasm32")]
19use std::cell::RefCell;
20use std::collections::{HashMap, HashSet, VecDeque};
21#[allow(unused_imports)]
22use std::sync::{
23    Arc,
24    atomic::{AtomicBool, AtomicU64, Ordering},
25};
26use std::time::Duration;
27#[cfg(not(target_arch = "wasm32"))]
28use std::time::{Instant, SystemTime, UNIX_EPOCH};
29#[cfg(not(target_arch = "wasm32"))]
30use tokio::sync::mpsc;
31#[cfg(not(target_arch = "wasm32"))]
32use tokio::task::JoinHandle as TokioJoinHandle;
33
34// FS persistence imports (native only when feature is enabled)
35#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
36use std::path::PathBuf;
37
38// Global storage management moved to vfs_sync module
39
40// Persistent metadata storage moved to metadata module
41
42// Auto-sync messaging
43#[cfg(not(target_arch = "wasm32"))]
44#[derive(Debug)]
45pub(super) enum SyncRequest {
46    Timer(tokio::sync::oneshot::Sender<()>),
47    Debounce(tokio::sync::oneshot::Sender<()>),
48}
49
50#[derive(Clone, Debug, Default)]
51pub struct RecoveryOptions {
52    pub mode: RecoveryMode,
53    pub on_corruption: CorruptionAction,
54}
55
56#[derive(Clone, Debug, Default)]
57pub enum RecoveryMode {
58    #[default]
59    Full,
60    Sample {
61        count: usize,
62    },
63    Skip,
64}
65
66#[derive(Clone, Debug, Default)]
67pub enum CorruptionAction {
68    #[default]
69    Report,
70    Repair,
71    Fail,
72}
73
74#[derive(Clone, Debug, PartialEq)]
75pub enum CrashRecoveryAction {
76    NoActionNeeded,
77    Rollback,
78    Finalize,
79}
80
81#[derive(Clone, Debug, Default)]
82pub struct RecoveryReport {
83    pub total_blocks_verified: usize,
84    pub corrupted_blocks: Vec<u64>,
85    pub repaired_blocks: Vec<u64>,
86    pub verification_duration_ms: u64,
87}
88
89// On-disk JSON schema for fs_persist
90#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
91#[derive(serde::Serialize, serde::Deserialize, Default)]
92#[allow(dead_code)]
93struct FsMeta {
94    entries: Vec<(u64, BlockMetadataPersist)>,
95}
96
97#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
98#[derive(serde::Serialize, serde::Deserialize, Default)]
99#[allow(dead_code)]
100struct FsAlloc {
101    allocated: Vec<u64>,
102}
103
104#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
105#[derive(serde::Serialize, serde::Deserialize, Default)]
106#[allow(dead_code)]
107struct FsDealloc {
108    tombstones: Vec<u64>,
109}
110
111// Metadata mirror for native builds
112#[cfg(not(target_arch = "wasm32"))]
113thread_local! {
114    pub(super) static GLOBAL_METADATA_TEST: parking_lot::Mutex<HashMap<String, HashMap<u64, BlockMetadataPersist>>> = parking_lot::Mutex::new(HashMap::new());
115}
116
117// Commit marker mirror for native builds (when fs_persist is disabled)
118#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
119thread_local! {
120    static GLOBAL_COMMIT_MARKER_TEST: parking_lot::Mutex<HashMap<String, u64>> = parking_lot::Mutex::new(HashMap::new());
121}
122
123#[derive(Clone, Debug)]
124pub struct SyncPolicy {
125    pub interval_ms: Option<u64>,
126    pub max_dirty: Option<usize>,
127    pub max_dirty_bytes: Option<usize>,
128    pub debounce_ms: Option<u64>,
129    pub verify_after_write: bool,
130}
131
132#[cfg(not(target_arch = "wasm32"))]
133impl Drop for BlockStorage {
134    fn drop(&mut self) {
135        if let Some(stop) = &self.auto_sync_stop {
136            stop.store(true, Ordering::SeqCst);
137        }
138        if let Some(handle) = self.auto_sync_thread.take() {
139            let _ = handle.join();
140        }
141        if let Some(handle) = self.debounce_thread.take() {
142            let _ = handle.join();
143        }
144        if let Some(task) = self.tokio_timer_task.take() {
145            task.abort();
146        }
147        if let Some(task) = self.tokio_debounce_task.take() {
148            task.abort();
149        }
150        self.auto_sync_stop = None;
151    }
152}
153
154pub const BLOCK_SIZE: usize = 4096;
155#[allow(dead_code)]
156pub(super) const DEFAULT_CACHE_CAPACITY: usize = 128;
157#[allow(dead_code)]
158const STORE_NAME: &str = "sqlite_blocks";
159#[allow(dead_code)]
160const METADATA_STORE: &str = "metadata";
161
162// Helper macro to abstract lock API differences: RefCell for WASM, Mutex for native
163// WASM: Use try_borrow_mut and handle reentrancy at call sites
164#[cfg(target_arch = "wasm32")]
165macro_rules! lock_mutex {
166    ($mutex:expr) => {
167        $mutex
168            .try_borrow_mut()
169            .expect("RefCell borrow failed - reentrancy detected in block_storage.rs")
170    };
171}
172
173// Try-lock version for reentrancy-safe operations
174#[allow(unused_macros)]
175#[cfg(target_arch = "wasm32")]
176macro_rules! try_lock_mutex {
177    ($mutex:expr) => {
178        $mutex
179    };
180}
181
182#[cfg(not(target_arch = "wasm32"))]
183macro_rules! lock_mutex {
184    ($mutex:expr) => {
185        $mutex.lock()
186    };
187}
188
189// Safe read macro for WASM that handles reentrancy
190#[allow(unused_macros)]
191#[cfg(target_arch = "wasm32")]
192macro_rules! read_lock {
193    ($mutex:expr) => {
194        $mutex
195            .try_borrow()
196            .expect("RefCell borrow failed - likely reentrancy issue")
197    };
198}
199
200#[allow(unused_macros)]
201#[cfg(not(target_arch = "wasm32"))]
202macro_rules! read_lock {
203    ($mutex:expr) => {
204        $mutex.lock()
205    };
206}
207
208// Try-lock macro for reentrancy-safe operations
209#[allow(unused_macros)]
210#[cfg(target_arch = "wasm32")]
211macro_rules! try_lock_mutex {
212    ($mutex:expr) => {
213        $mutex.ok()
214    };
215}
216
217#[allow(unused_macros)]
218#[cfg(not(target_arch = "wasm32"))]
219macro_rules! try_lock_mutex {
220    ($mutex:expr) => {
221        Some($mutex.lock())
222    };
223}
224
225// Try-read macro for reentrancy-safe read operations
226#[allow(unused_macros)]
227#[cfg(target_arch = "wasm32")]
228macro_rules! try_read_lock {
229    ($mutex:expr) => {
230        $mutex.try_borrow().ok()
231    };
232}
233
234#[allow(unused_macros)]
235#[cfg(not(target_arch = "wasm32"))]
236macro_rules! try_read_lock {
237    ($mutex:expr) => {
238        Some($mutex.lock())
239    };
240}
241
242#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
243pub enum StorageBackend {
244    #[default]
245    IndexedDb,
246    Opfs,
247    Hybrid,
248}
249
250impl StorageBackend {
251    pub fn as_str(self) -> &'static str {
252        match self {
253            StorageBackend::IndexedDb => "IndexedDB",
254            StorageBackend::Opfs => "OPFS",
255            StorageBackend::Hybrid => "Hybrid",
256        }
257    }
258}
259
260pub struct BlockStorage {
261    // WASM: RefCell for zero-cost interior mutability (single-threaded)
262    // Native: Mutex for thread safety
263    #[cfg(target_arch = "wasm32")]
264    pub(super) cache: RefCell<HashMap<u64, Vec<u8>>>,
265    #[cfg(not(target_arch = "wasm32"))]
266    pub(super) cache: Mutex<HashMap<u64, Vec<u8>>>,
267
268    #[cfg(target_arch = "wasm32")]
269    pub(super) dirty_blocks: Arc<RefCell<HashMap<u64, Vec<u8>>>>,
270    #[cfg(not(target_arch = "wasm32"))]
271    pub(super) dirty_blocks: Arc<Mutex<HashMap<u64, Vec<u8>>>>,
272
273    #[cfg(target_arch = "wasm32")]
274    pub(super) allocated_blocks: RefCell<HashSet<u64>>,
275    #[cfg(not(target_arch = "wasm32"))]
276    pub(super) allocated_blocks: Mutex<HashSet<u64>>,
277
278    #[cfg(target_arch = "wasm32")]
279    #[allow(dead_code)]
280    pub(super) deallocated_blocks: RefCell<HashSet<u64>>,
281    #[cfg(not(target_arch = "wasm32"))]
282    #[allow(dead_code)]
283    pub(super) deallocated_blocks: Mutex<HashSet<u64>>,
284    pub(super) next_block_id: AtomicU64,
285    pub(super) capacity: usize,
286
287    #[cfg(target_arch = "wasm32")]
288    pub(super) lru_order: RefCell<VecDeque<u64>>,
289    #[cfg(not(target_arch = "wasm32"))]
290    pub(super) lru_order: Mutex<VecDeque<u64>>,
291
292    // Checksum management (moved to metadata module)
293    pub(super) checksum_manager: ChecksumManager,
294    #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
295    pub(super) base_dir: PathBuf,
296    pub(super) db_name: String,
297    pub(super) backend: StorageBackend,
298
299    // Background sync settings
300    #[cfg(target_arch = "wasm32")]
301    pub(super) auto_sync_interval: RefCell<Option<Duration>>,
302    #[cfg(not(target_arch = "wasm32"))]
303    pub(super) auto_sync_interval: Mutex<Option<Duration>>,
304
305    #[cfg(not(target_arch = "wasm32"))]
306    pub(super) last_auto_sync: Instant,
307
308    #[cfg(target_arch = "wasm32")]
309    pub(super) policy: RefCell<Option<SyncPolicy>>,
310    #[cfg(not(target_arch = "wasm32"))]
311    pub(super) policy: Mutex<Option<SyncPolicy>>,
312    #[cfg(not(target_arch = "wasm32"))]
313    pub(super) auto_sync_stop: Option<Arc<AtomicBool>>,
314    #[cfg(not(target_arch = "wasm32"))]
315    pub(super) auto_sync_thread: Option<std::thread::JoinHandle<()>>,
316    #[cfg(not(target_arch = "wasm32"))]
317    pub(super) debounce_thread: Option<std::thread::JoinHandle<()>>,
318    #[cfg(not(target_arch = "wasm32"))]
319    pub(super) tokio_timer_task: Option<TokioJoinHandle<()>>,
320    #[cfg(not(target_arch = "wasm32"))]
321    pub(super) tokio_debounce_task: Option<TokioJoinHandle<()>>,
322    #[cfg(not(target_arch = "wasm32"))]
323    pub(super) last_write_ms: Arc<AtomicU64>,
324    #[cfg(not(target_arch = "wasm32"))]
325    pub(super) threshold_hit: Arc<AtomicBool>,
326    #[cfg(not(target_arch = "wasm32"))]
327    pub(super) sync_count: Arc<AtomicU64>,
328    #[cfg(not(target_arch = "wasm32"))]
329    pub(super) timer_sync_count: Arc<AtomicU64>,
330    #[cfg(not(target_arch = "wasm32"))]
331    pub(super) debounce_sync_count: Arc<AtomicU64>,
332    #[cfg(not(target_arch = "wasm32"))]
333    pub(super) last_sync_duration_ms: Arc<AtomicU64>,
334
335    // Auto-sync channel for real sync operations
336    #[cfg(not(target_arch = "wasm32"))]
337    pub(super) sync_sender: Option<mpsc::UnboundedSender<SyncRequest>>,
338    #[cfg(not(target_arch = "wasm32"))]
339    pub(super) sync_receiver: Option<mpsc::UnboundedReceiver<SyncRequest>>,
340
341    // Startup recovery report
342    pub(super) recovery_report: RecoveryReport,
343
344    // Leader election manager (WASM only) - wrapped in RefCell for interior mutability
345    #[cfg(target_arch = "wasm32")]
346    pub leader_election: std::cell::RefCell<Option<super::leader_election::LeaderElectionManager>>,
347
348    // Observability manager
349    pub(super) observability: super::observability::ObservabilityManager,
350
351    // Telemetry metrics (optional)
352    #[cfg(feature = "telemetry")]
353    pub(super) metrics: Option<crate::telemetry::Metrics>,
354}
355
356impl BlockStorage {
357    /// Create a new BlockStorage synchronously without IndexedDB restoration
358    /// Used for auto-registration in VFS when existing data is detected
359    #[cfg(target_arch = "wasm32")]
360    pub fn new_sync(db_name: &str) -> Self {
361        log::info!(
362            "Creating BlockStorage synchronously for database: {}",
363            db_name
364        );
365
366        // Load existing data from GLOBAL_STORAGE to support multi-connection scenarios
367        use crate::storage::vfs_sync::with_global_storage;
368        let (cache, allocated_blocks, max_block_id) = with_global_storage(|storage_map| {
369            if let Some(db_storage) = storage_map.borrow().get(db_name) {
370                let cache = db_storage.clone();
371                let allocated = db_storage.keys().copied().collect::<HashSet<_>>();
372                let max_id = db_storage.keys().max().copied().unwrap_or(0);
373                (cache, allocated, max_id)
374            } else {
375                (HashMap::new(), HashSet::new(), 0)
376            }
377        });
378
379        log::info!(
380            "Loaded {} blocks from GLOBAL_STORAGE for {} (max_block_id={})",
381            cache.len(),
382            db_name,
383            max_block_id
384        );
385
386        // CRITICAL: Always reload checksums from GLOBAL_METADATA since cache might be stale
387        // This handles the case where import writes new data after close() reloaded cache
388        use crate::storage::vfs_sync::with_global_metadata;
389        let checksum_manager = with_global_metadata(|metadata_map| {
390            if let Some(db_metadata) = metadata_map.borrow().get(db_name) {
391                let mut checksums = HashMap::new();
392                let mut algos = HashMap::new();
393                for (block_id, meta) in db_metadata {
394                    checksums.insert(*block_id, meta.checksum);
395                    algos.insert(*block_id, meta.algo);
396                }
397                ChecksumManager::with_data(checksums, algos, ChecksumAlgorithm::FastHash)
398            } else {
399                ChecksumManager::new(ChecksumAlgorithm::FastHash)
400            }
401        });
402
403        Self {
404            cache: RefCell::new(cache),
405            dirty_blocks: Arc::new(RefCell::new(HashMap::new())),
406            allocated_blocks: RefCell::new(allocated_blocks),
407            deallocated_blocks: RefCell::new(HashSet::new()),
408            next_block_id: AtomicU64::new(max_block_id + 1),
409            capacity: 128,
410            lru_order: RefCell::new(VecDeque::new()),
411            checksum_manager,
412            db_name: db_name.to_string(),
413            backend: StorageBackend::IndexedDb,
414            auto_sync_interval: RefCell::new(None),
415            policy: RefCell::new(None),
416            #[cfg(not(target_arch = "wasm32"))]
417            last_auto_sync: Instant::now(),
418            #[cfg(not(target_arch = "wasm32"))]
419            auto_sync_stop: None,
420            #[cfg(not(target_arch = "wasm32"))]
421            auto_sync_thread: None,
422            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
423            base_dir: std::path::PathBuf::from(
424                std::env::var("ABSURDERSQL_FS_BASE")
425                    .unwrap_or_else(|_| "./test_storage".to_string()),
426            ),
427            #[cfg(not(target_arch = "wasm32"))]
428            debounce_thread: None,
429            #[cfg(not(target_arch = "wasm32"))]
430            tokio_timer_task: None,
431            #[cfg(not(target_arch = "wasm32"))]
432            tokio_debounce_task: None,
433            #[cfg(not(target_arch = "wasm32"))]
434            last_write_ms: Arc::new(AtomicU64::new(0)),
435            #[cfg(not(target_arch = "wasm32"))]
436            threshold_hit: Arc::new(AtomicBool::new(false)),
437            #[cfg(not(target_arch = "wasm32"))]
438            sync_count: Arc::new(AtomicU64::new(0)),
439            #[cfg(not(target_arch = "wasm32"))]
440            timer_sync_count: Arc::new(AtomicU64::new(0)),
441            #[cfg(not(target_arch = "wasm32"))]
442            debounce_sync_count: Arc::new(AtomicU64::new(0)),
443            #[cfg(not(target_arch = "wasm32"))]
444            last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
445            #[cfg(not(target_arch = "wasm32"))]
446            sync_sender: None,
447            #[cfg(not(target_arch = "wasm32"))]
448            sync_receiver: None,
449            recovery_report: RecoveryReport::default(),
450            #[cfg(target_arch = "wasm32")]
451            leader_election: std::cell::RefCell::new(None),
452            observability: super::observability::ObservabilityManager::new(),
453            #[cfg(feature = "telemetry")]
454            metrics: None,
455        }
456    }
457
458    #[cfg(target_arch = "wasm32")]
459    pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
460        super::constructors::new_wasm(db_name).await
461    }
462
463    #[cfg(target_arch = "wasm32")]
464    pub async fn new_with_backend(
465        db_name: &str,
466        backend: StorageBackend,
467    ) -> Result<Self, DatabaseError> {
468        super::constructors::new_wasm_with_backend(db_name, backend).await
469    }
470
471    #[cfg(not(target_arch = "wasm32"))]
472    pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
473        // CRITICAL: Use centralized normalize_db_name to match VFS and Database.name
474        let db_name = crate::utils::normalize_db_name(db_name);
475        let db_name = db_name.as_str();
476        log::info!("Creating BlockStorage for database: {}", db_name);
477
478        // Initialize allocation tracking for native
479        let (allocated_blocks, next_block_id) = {
480            #[cfg(feature = "fs_persist")]
481            {
482                // fs_persist: restore allocation from filesystem
483                let mut allocated_blocks = HashSet::new();
484                let mut next_block_id: u64 = 1;
485
486                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
487                    .unwrap_or_else(|_| "./test_storage".to_string());
488                let mut alloc_path = std::path::PathBuf::from(base_path);
489                alloc_path.push(db_name);
490                alloc_path.push("allocations.json");
491
492                if let Ok(content) = std::fs::read_to_string(&alloc_path) {
493                    if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
494                        if let Some(allocated_array) = alloc_data["allocated"].as_array() {
495                            for block_id_val in allocated_array {
496                                if let Some(block_id) = block_id_val.as_u64() {
497                                    allocated_blocks.insert(block_id);
498                                }
499                            }
500                            next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
501                        }
502                    }
503                }
504
505                (allocated_blocks, next_block_id)
506            }
507
508            #[cfg(not(feature = "fs_persist"))]
509            {
510                // Native non-fs_persist: restore allocation from global storage
511                let mut allocated_blocks = HashSet::new();
512                let mut next_block_id: u64 = 1;
513
514                super::vfs_sync::with_global_allocation_map(|allocation_map| {
515                    if let Some(existing_allocations) = allocation_map.borrow().get(db_name) {
516                        allocated_blocks = existing_allocations.clone();
517                        next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
518                        log::info!(
519                            "Restored {} allocated blocks for database: {}",
520                            allocated_blocks.len(),
521                            db_name
522                        );
523                    }
524                });
525
526                (allocated_blocks, next_block_id)
527            }
528        };
529
530        // Initialize checksums and checksum algorithms
531        let checksums_init: HashMap<u64, u64> = {
532            #[cfg(feature = "fs_persist")]
533            {
534                let mut map = HashMap::new();
535                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
536                    .unwrap_or_else(|_| "./test_storage".to_string());
537                let mut meta_path = std::path::PathBuf::from(base_path);
538                meta_path.push(db_name);
539                meta_path.push("metadata.json");
540
541                if let Ok(content) = std::fs::read_to_string(&meta_path) {
542                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
543                        if let Some(entries) = meta_data["entries"].as_array() {
544                            for entry in entries {
545                                if let Some(arr) = entry.as_array() {
546                                    if let (Some(block_id), Some(obj)) = (
547                                        arr.first().and_then(|v| v.as_u64()),
548                                        arr.get(1).and_then(|v| v.as_object()),
549                                    ) {
550                                        if let Some(checksum) =
551                                            obj.get("checksum").and_then(|v| v.as_u64())
552                                        {
553                                            map.insert(block_id, checksum);
554                                        }
555                                    }
556                                }
557                            }
558                        }
559                    }
560                }
561                map
562            }
563
564            #[cfg(not(feature = "fs_persist"))]
565            {
566                // Native non-fs_persist: restore checksums from global test storage
567                let mut map = HashMap::new();
568                GLOBAL_METADATA_TEST.with(|meta| {
569                    let meta_map = meta.lock();
570                    if let Some(db_meta) = meta_map.get(db_name) {
571                        for (block_id, metadata) in db_meta.iter() {
572                            map.insert(*block_id, metadata.checksum);
573                        }
574                    }
575                });
576                map
577            }
578        };
579
580        let checksum_algos_init: HashMap<u64, ChecksumAlgorithm> = {
581            #[cfg(feature = "fs_persist")]
582            {
583                let mut map = HashMap::new();
584                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
585                    .unwrap_or_else(|_| "./test_storage".to_string());
586                let mut meta_path = std::path::PathBuf::from(base_path);
587                meta_path.push(db_name);
588                meta_path.push("metadata.json");
589
590                if let Ok(content) = std::fs::read_to_string(&meta_path) {
591                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
592                        if let Some(entries) = meta_data["entries"].as_array() {
593                            for entry in entries {
594                                if let Some(arr) = entry.as_array() {
595                                    if let (Some(block_id), Some(obj)) = (
596                                        arr.first().and_then(|v| v.as_u64()),
597                                        arr.get(1).and_then(|v| v.as_object()),
598                                    ) {
599                                        let algo = obj
600                                            .get("algo")
601                                            .and_then(|v| v.as_str())
602                                            .and_then(|s| match s {
603                                                "CRC32" => Some(ChecksumAlgorithm::CRC32),
604                                                "FastHash" => Some(ChecksumAlgorithm::FastHash),
605                                                _ => None,
606                                            })
607                                            .unwrap_or(ChecksumAlgorithm::FastHash);
608                                        map.insert(block_id, algo);
609                                    }
610                                }
611                            }
612                        }
613                    }
614                }
615                map
616            }
617
618            #[cfg(not(feature = "fs_persist"))]
619            {
620                // Native non-fs_persist: restore algorithms from global test storage
621                let mut map = HashMap::new();
622                GLOBAL_METADATA_TEST.with(|meta| {
623                    let meta_map = meta.lock();
624                    if let Some(db_meta) = meta_map.get(db_name) {
625                        for (block_id, metadata) in db_meta.iter() {
626                            map.insert(*block_id, metadata.algo);
627                        }
628                    }
629                });
630                map
631            }
632        };
633
634        // Determine default checksum algorithm from environment (fs_persist native), fallback to FastHash
635        #[cfg(feature = "fs_persist")]
636        let checksum_algo_default = match std::env::var("DATASYNC_CHECKSUM_ALGO").ok().as_deref() {
637            Some("CRC32") => ChecksumAlgorithm::CRC32,
638            _ => ChecksumAlgorithm::FastHash,
639        };
640        #[cfg(not(feature = "fs_persist"))]
641        let checksum_algo_default = ChecksumAlgorithm::FastHash;
642
643        // Load deallocated blocks from filesystem
644        let deallocated_blocks_init: HashSet<u64> = {
645            #[cfg(feature = "fs_persist")]
646            {
647                let mut set = HashSet::new();
648                let base_path = std::env::var("ABSURDERSQL_FS_BASE")
649                    .unwrap_or_else(|_| "./test_storage".to_string());
650                let mut path = std::path::PathBuf::from(base_path);
651                path.push(db_name);
652                let mut dealloc_path = path.clone();
653                dealloc_path.push("deallocated.json");
654                if let Ok(content) = std::fs::read_to_string(&dealloc_path) {
655                    if let Ok(dealloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
656                        if let Some(tombstones_array) = dealloc_data["tombstones"].as_array() {
657                            for tombstone_val in tombstones_array {
658                                if let Some(block_id) = tombstone_val.as_u64() {
659                                    set.insert(block_id);
660                                }
661                            }
662                        }
663                    }
664                }
665                set
666            }
667            #[cfg(not(feature = "fs_persist"))]
668            {
669                HashSet::new()
670            }
671        };
672
673        Ok(BlockStorage {
674            db_name: db_name.to_string(),
675            backend: StorageBackend::IndexedDb,
676            cache: Mutex::new(HashMap::new()),
677            lru_order: Mutex::new(VecDeque::new()),
678            capacity: 1000,
679            checksum_manager: ChecksumManager::with_data(
680                checksums_init,
681                checksum_algos_init,
682                checksum_algo_default,
683            ),
684            dirty_blocks: Arc::new(Mutex::new(HashMap::new())),
685            allocated_blocks: Mutex::new(allocated_blocks),
686            next_block_id: AtomicU64::new(next_block_id),
687            deallocated_blocks: Mutex::new(deallocated_blocks_init),
688            policy: Mutex::new(None),
689            auto_sync_interval: Mutex::new(None),
690            #[cfg(not(target_arch = "wasm32"))]
691            last_auto_sync: Instant::now(),
692            #[cfg(not(target_arch = "wasm32"))]
693            auto_sync_stop: None,
694            #[cfg(not(target_arch = "wasm32"))]
695            auto_sync_thread: None,
696            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
697            base_dir: std::path::PathBuf::from(
698                std::env::var("ABSURDERSQL_FS_BASE")
699                    .unwrap_or_else(|_| "./test_storage".to_string()),
700            ),
701            #[cfg(not(target_arch = "wasm32"))]
702            debounce_thread: None,
703            #[cfg(not(target_arch = "wasm32"))]
704            tokio_timer_task: None,
705            #[cfg(not(target_arch = "wasm32"))]
706            tokio_debounce_task: None,
707            #[cfg(not(target_arch = "wasm32"))]
708            last_write_ms: Arc::new(AtomicU64::new(0)),
709            #[cfg(not(target_arch = "wasm32"))]
710            threshold_hit: Arc::new(AtomicBool::new(false)),
711            #[cfg(not(target_arch = "wasm32"))]
712            sync_count: Arc::new(AtomicU64::new(0)),
713            #[cfg(not(target_arch = "wasm32"))]
714            timer_sync_count: Arc::new(AtomicU64::new(0)),
715            #[cfg(not(target_arch = "wasm32"))]
716            debounce_sync_count: Arc::new(AtomicU64::new(0)),
717            #[cfg(not(target_arch = "wasm32"))]
718            last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
719            #[cfg(not(target_arch = "wasm32"))]
720            sync_sender: None,
721            #[cfg(not(target_arch = "wasm32"))]
722            sync_receiver: None,
723            recovery_report: RecoveryReport::default(),
724            #[cfg(target_arch = "wasm32")]
725            leader_election: std::cell::RefCell::new(None),
726            observability: super::observability::ObservabilityManager::new(),
727            #[cfg(feature = "telemetry")]
728            metrics: None,
729        })
730    }
731
732    pub async fn new_with_capacity(db_name: &str, capacity: usize) -> Result<Self, DatabaseError> {
733        let mut s = Self::new(db_name).await?;
734        s.capacity = capacity;
735        Ok(s)
736    }
737
738    pub fn storage_backend(&self) -> StorageBackend {
739        self.backend
740    }
741
742    pub async fn new_with_recovery_options(
743        db_name: &str,
744        recovery_opts: RecoveryOptions,
745    ) -> Result<Self, DatabaseError> {
746        let mut storage = Self::new(db_name).await?;
747
748        storage.perform_startup_recovery(recovery_opts).await?;
749
750        Ok(storage)
751    }
752
753    pub fn get_recovery_report(&self) -> &RecoveryReport {
754        &self.recovery_report
755    }
756
757    async fn perform_startup_recovery(
758        &mut self,
759        opts: RecoveryOptions,
760    ) -> Result<(), DatabaseError> {
761        super::recovery::perform_startup_recovery(self, opts).await
762    }
763
764    pub(super) async fn get_blocks_for_verification(
765        &self,
766        mode: &RecoveryMode,
767    ) -> Result<Vec<u64>, DatabaseError> {
768        let all_blocks: Vec<u64> = lock_mutex!(self.allocated_blocks).iter().copied().collect();
769
770        match mode {
771            RecoveryMode::Full => Ok(all_blocks),
772            RecoveryMode::Sample { count } => {
773                let sample_count = (*count).min(all_blocks.len());
774                let mut sampled = all_blocks;
775                sampled.sort_unstable(); // Deterministic sampling
776                sampled.truncate(sample_count);
777                Ok(sampled)
778            }
779            RecoveryMode::Skip => Ok(Vec::new()),
780        }
781    }
782
783    pub(super) async fn verify_block_integrity(
784        &mut self,
785        block_id: u64,
786    ) -> Result<bool, DatabaseError> {
787        // Read the block data
788        let data = match self.read_block_from_storage(block_id).await {
789            Ok(data) => data,
790            Err(_) => {
791                log::warn!(
792                    "Could not read block {} for integrity verification",
793                    block_id
794                );
795                return Ok(false);
796            }
797        };
798
799        // Verify against stored checksum
800        match self.verify_against_stored_checksum(block_id, &data) {
801            Ok(()) => Ok(true),
802            Err(e) => {
803                log::warn!(
804                    "Block {} failed checksum verification: {}",
805                    block_id,
806                    e.message
807                );
808                Ok(false)
809            }
810        }
811    }
812
813    async fn read_block_from_storage(&mut self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
814        // Try to read from filesystem first (fs_persist mode)
815        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
816        {
817            let mut blocks_dir = self.base_dir.clone();
818            blocks_dir.push(&self.db_name);
819            blocks_dir.push("blocks");
820            let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
821
822            if let Ok(data) = std::fs::read(&block_file) {
823                if data.len() == BLOCK_SIZE {
824                    return Ok(data);
825                }
826            }
827        }
828
829        // Fallback to test storage for native tests
830        #[cfg(all(
831            not(target_arch = "wasm32"),
832            any(test, debug_assertions),
833            not(feature = "fs_persist")
834        ))]
835        {
836            let mut found_data = None;
837            vfs_sync::with_global_storage(|storage| {
838                #[cfg(target_arch = "wasm32")]
839                let storage_map = storage;
840                #[cfg(not(target_arch = "wasm32"))]
841                let storage_map = storage.borrow();
842                if let Some(db_storage) = storage_map.get(&self.db_name) {
843                    if let Some(data) = db_storage.get(&block_id) {
844                        found_data = Some(data.clone());
845                    }
846                }
847            });
848            if let Some(data) = found_data {
849                return Ok(data);
850            }
851        }
852
853        // WASM global storage
854        #[cfg(target_arch = "wasm32")]
855        {
856            let mut found_data = None;
857            vfs_sync::with_global_storage(|storage_map| {
858                if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
859                    if let Some(data) = db_storage.get(&block_id) {
860                        found_data = Some(data.clone());
861                    }
862                }
863            });
864            if let Some(data) = found_data {
865                return Ok(data);
866            }
867        }
868
869        Err(DatabaseError::new(
870            "BLOCK_NOT_FOUND",
871            &format!("Block {} not found in storage", block_id),
872        ))
873    }
874
875    pub(super) async fn repair_corrupted_block(
876        &mut self,
877        block_id: u64,
878    ) -> Result<bool, DatabaseError> {
879        log::info!("Attempting to repair corrupted block {}", block_id);
880
881        // For now, repair by removing the corrupted block and clearing its metadata
882        // In a real implementation, this might involve restoring from backup or rebuilding
883
884        // Remove from cache
885        lock_mutex!(self.cache).remove(&block_id);
886
887        // Remove checksum metadata
888        self.checksum_manager.remove_checksum(block_id);
889
890        // Remove from filesystem if fs_persist is enabled
891        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
892        {
893            let mut blocks_dir = self.base_dir.clone();
894            blocks_dir.push(&self.db_name);
895            blocks_dir.push("blocks");
896            let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
897            let _ = std::fs::remove_file(&block_file);
898        }
899
900        // Remove from test storage
901        #[cfg(all(
902            not(target_arch = "wasm32"),
903            any(test, debug_assertions),
904            not(feature = "fs_persist")
905        ))]
906        {
907            vfs_sync::with_global_storage(|storage| {
908                let mut storage_map = storage.borrow_mut();
909                if let Some(db_storage) = storage_map.get_mut(&self.db_name) {
910                    db_storage.remove(&block_id);
911                }
912            });
913        }
914
915        // Remove from WASM storage
916        #[cfg(target_arch = "wasm32")]
917        {
918            vfs_sync::with_global_storage(|storage_map| {
919                if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
920                    db_storage.remove(&block_id);
921                }
922            });
923        }
924
925        log::info!(
926            "Corrupted block {} has been removed (repair completed)",
927            block_id
928        );
929        Ok(true)
930    }
931
932    pub(super) fn touch_lru(&self, block_id: u64) {
933        // Remove any existing occurrence
934        let mut lru = lock_mutex!(self.lru_order);
935        if let Some(pos) = lru.iter().position(|&id| id == block_id) {
936            lru.remove(pos);
937        }
938        // Push as most-recent
939        lru.push_back(block_id);
940    }
941
942    pub(super) fn evict_if_needed(&self) {
943        // Evict clean LRU blocks until within capacity. Never evict dirty blocks.
944        loop {
945            // Check capacity and find victim in a single critical section
946            let victim_opt = {
947                let cache_guard = lock_mutex!(self.cache);
948                if cache_guard.len() <= self.capacity {
949                    break; // Within capacity, done
950                }
951
952                // Find the least-recent block that is NOT dirty
953                let dirty_guard = lock_mutex!(self.dirty_blocks);
954                let mut lru_guard = lock_mutex!(self.lru_order);
955
956                let victim_pos = lru_guard
957                    .iter()
958                    .position(|id| !dirty_guard.contains_key(id));
959
960                victim_pos.map(|pos| lru_guard.remove(pos).expect("valid pos"))
961                // Guards are dropped here
962            };
963
964            match victim_opt {
965                Some(victim) => {
966                    // Remove from cache (new lock acquisition)
967                    lock_mutex!(self.cache).remove(&victim);
968                }
969                None => {
970                    // All blocks are dirty; cannot evict. Allow temporary overflow.
971                    break;
972                }
973            }
974        }
975    }
976
977    #[inline]
978    #[cfg(target_arch = "wasm32")]
979    pub fn now_millis() -> u64 {
980        // Date::now() returns milliseconds since UNIX epoch as f64
981        Date::now() as u64
982    }
983
984    #[inline]
985    #[cfg(not(target_arch = "wasm32"))]
986    pub fn now_millis() -> u64 {
987        let now = SystemTime::now()
988            .duration_since(UNIX_EPOCH)
989            .unwrap_or_else(|_| Duration::from_millis(0));
990        now.as_millis() as u64
991    }
992
993    pub(super) fn verify_against_stored_checksum(
994        &self,
995        block_id: u64,
996        data: &[u8],
997    ) -> Result<(), DatabaseError> {
998        self.checksum_manager.validate_checksum(block_id, data)
999    }
1000
1001    /// Synchronous block read for environments that require sync access (e.g., VFS callbacks)
1002    pub fn read_block_sync(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
1003        // Implementation moved to io_operations module
1004        super::io_operations::read_block_sync_impl(self, block_id)
1005    }
1006
1007    pub async fn read_block(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
1008        // Delegate to synchronous implementation (immediately ready)
1009        self.read_block_sync(block_id)
1010    }
1011
1012    /// Synchronous block write for environments that require sync access (e.g., VFS callbacks)
1013    #[cfg(target_arch = "wasm32")]
1014    pub fn write_block_sync(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1015        super::io_operations::write_block_sync_impl(self, block_id, data)
1016    }
1017
1018    #[cfg(not(target_arch = "wasm32"))]
1019    pub fn write_block_sync(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1020        super::io_operations::write_block_sync_impl(self, block_id, data)
1021    }
1022
1023    #[cfg(target_arch = "wasm32")]
1024    pub async fn write_block(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1025        self.write_block_sync(block_id, data)
1026    }
1027
1028    #[cfg(not(target_arch = "wasm32"))]
1029    pub async fn write_block(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1030        self.write_block_sync(block_id, data)?;
1031
1032        // If threshold was hit and debounce is NOT enabled, perform inline sync
1033        if self.threshold_hit.load(std::sync::atomic::Ordering::SeqCst) {
1034            let has_debounce = lock_mutex!(self.policy)
1035                .as_ref()
1036                .and_then(|p| p.debounce_ms)
1037                .is_some();
1038            if !has_debounce {
1039                log::debug!("Threshold hit without debounce: performing inline sync");
1040                self.sync_implementation()?;
1041                self.threshold_hit
1042                    .store(false, std::sync::atomic::Ordering::SeqCst);
1043            }
1044        }
1045
1046        Ok(())
1047    }
1048
1049    /// Synchronous batch write of blocks
1050    #[cfg(target_arch = "wasm32")]
1051    pub fn write_blocks_sync(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1052        self.maybe_auto_sync();
1053        for (block_id, data) in items {
1054            self.write_block_sync(block_id, data)?;
1055        }
1056        Ok(())
1057    }
1058
1059    #[cfg(not(target_arch = "wasm32"))]
1060    pub fn write_blocks_sync(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1061        self.maybe_auto_sync();
1062        for (block_id, data) in items {
1063            self.write_block_sync(block_id, data)?;
1064        }
1065        Ok(())
1066    }
1067
1068    /// Async batch write wrapper
1069    #[cfg(target_arch = "wasm32")]
1070    pub async fn write_blocks(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1071        self.write_blocks_sync(items)
1072    }
1073
1074    #[cfg(not(target_arch = "wasm32"))]
1075    pub async fn write_blocks(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1076        self.write_blocks_sync(items)
1077    }
1078
1079    /// Synchronous batch read of blocks, preserving input order
1080    pub fn read_blocks_sync(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1081        self.maybe_auto_sync();
1082        let mut results = Vec::with_capacity(block_ids.len());
1083        for &id in block_ids {
1084            results.push(self.read_block_sync(id)?);
1085        }
1086        Ok(results)
1087    }
1088
1089    /// Async batch read wrapper
1090    pub async fn read_blocks(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1091        self.read_blocks_sync(block_ids)
1092    }
1093
1094    /// Get block checksum for verification
1095    pub fn get_block_checksum(&self, block_id: u64) -> Option<u32> {
1096        self.checksum_manager
1097            .get_checksum(block_id)
1098            .map(|checksum| checksum as u32)
1099    }
1100
1101    /// Get current commit marker for this database (WASM only, for testing)
1102    #[cfg(target_arch = "wasm32")]
1103    pub fn get_commit_marker(&self) -> u64 {
1104        vfs_sync::with_global_commit_marker(|cm| {
1105            cm.borrow().get(&self.db_name).copied().unwrap_or(0)
1106        })
1107    }
1108
1109    /// Check if this database has any blocks in storage (WASM only)
1110    #[cfg(target_arch = "wasm32")]
1111    pub fn has_any_blocks(&self) -> bool {
1112        vfs_sync::with_global_storage(|storage_map| {
1113            storage_map
1114                .borrow()
1115                .get(&self.db_name)
1116                .map_or(false, |blocks| !blocks.is_empty())
1117        })
1118    }
1119
1120    pub async fn verify_block_checksum(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1121        // If cached, verify directly against cached bytes
1122        if let Some(bytes) = lock_mutex!(self.cache).get(&block_id).cloned() {
1123            return self.verify_against_stored_checksum(block_id, &bytes);
1124        }
1125        // Otherwise, a read will populate cache and also verify
1126        let data = self.read_block_sync(block_id)?;
1127        self.verify_against_stored_checksum(block_id, &data)
1128    }
1129
1130    // Always available for testing (integration tests need this in release mode)
1131    #[allow(unused_mut)]
1132    pub fn get_block_metadata_for_testing(&mut self) -> HashMap<u64, (u64, u32, u64)> {
1133        // Returns map of block_id -> (checksum, version, last_modified_ms)
1134        #[cfg(target_arch = "wasm32")]
1135        {
1136            let mut out = HashMap::new();
1137            vfs_sync::with_global_metadata(|meta_map| {
1138                if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1139                    for (bid, m) in db_meta.iter() {
1140                        out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1141                    }
1142                }
1143            });
1144            out
1145        }
1146        #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
1147        {
1148            use super::fs_persist::FsMeta;
1149            use std::io::Read;
1150            let mut out = HashMap::new();
1151            let base: PathBuf = self.base_dir.clone();
1152            let mut db_dir = base.clone();
1153            db_dir.push(&self.db_name);
1154            let mut meta_path = db_dir.clone();
1155            meta_path.push("metadata.json");
1156            if let Ok(mut f) = std::fs::File::open(&meta_path) {
1157                let mut s = String::new();
1158                if f.read_to_string(&mut s).is_ok() {
1159                    if let Ok(meta) = serde_json::from_str::<FsMeta>(&s) {
1160                        for (block_id, metadata) in meta.entries {
1161                            out.insert(
1162                                block_id,
1163                                (
1164                                    metadata.checksum,
1165                                    metadata.version,
1166                                    metadata.last_modified_ms,
1167                                ),
1168                            );
1169                        }
1170                    }
1171                }
1172            }
1173            out
1174        }
1175        #[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
1176        {
1177            let mut out = HashMap::new();
1178            GLOBAL_METADATA_TEST.with(|meta| {
1179                let meta_map = meta.lock();
1180                if let Some(db_meta) = meta_map.get(&self.db_name) {
1181                    for (bid, m) in db_meta.iter() {
1182                        out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1183                    }
1184                }
1185            });
1186            out
1187        }
1188    }
1189
1190    // Always available for testing (integration tests need this in release mode)
1191    pub fn set_block_checksum_for_testing(&mut self, block_id: u64, checksum: u64) {
1192        self.checksum_manager
1193            .set_checksum_for_testing(block_id, checksum);
1194    }
1195
1196    /// Getter for dirty_blocks for fs_persist and auto_sync modules
1197    #[cfg(not(target_arch = "wasm32"))]
1198    pub(super) fn get_dirty_blocks(&self) -> &Arc<Mutex<HashMap<u64, Vec<u8>>>> {
1199        &self.dirty_blocks
1200    }
1201
1202    #[cfg(target_arch = "wasm32")]
1203    pub async fn sync(&self) -> Result<(), DatabaseError> {
1204        self.sync_async().await
1205    }
1206
1207    #[cfg(not(target_arch = "wasm32"))]
1208    pub async fn sync(&mut self) -> Result<(), DatabaseError> {
1209        self.sync_implementation()
1210    }
1211
1212    /// Synchronous version of sync() for immediate persistence
1213    #[cfg(target_arch = "wasm32")]
1214    #[allow(invalid_reference_casting)]
1215    pub fn sync_now(&self) -> Result<(), DatabaseError> {
1216        // WASM can use &self due to RefCell interior mutability
1217        // Cast self to mutable for the implementation
1218        let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1219        self_mut.sync_implementation()
1220    }
1221
1222    #[cfg(not(target_arch = "wasm32"))]
1223    pub fn sync_now(&mut self) -> Result<(), DatabaseError> {
1224        self.sync_implementation()
1225    }
1226
1227    /// Internal sync implementation shared by sync() and sync_now()
1228    fn sync_implementation(&mut self) -> Result<(), DatabaseError> {
1229        super::sync_operations::sync_implementation_impl(self)
1230    }
1231
1232    /// Sync blocks to global storage without advancing commit marker
1233    /// Used by VFS x_sync callback to persist blocks but maintain commit marker lag
1234    #[cfg(target_arch = "wasm32")]
1235    pub fn sync_blocks_only(&mut self) -> Result<(), DatabaseError> {
1236        super::wasm_vfs_sync::sync_blocks_only(self)
1237    }
1238
1239    /// Async version of sync for WASM that properly awaits IndexedDB persistence
1240    #[cfg(target_arch = "wasm32")]
1241    pub async fn sync_async(&self) -> Result<(), DatabaseError> {
1242        super::wasm_indexeddb::sync_async(self).await
1243    }
1244
1245    /// Drain all pending dirty blocks and stop background auto-sync (if enabled).
1246    /// Safe to call multiple times.
1247    #[cfg(target_arch = "wasm32")]
1248    #[allow(invalid_reference_casting)]
1249    pub fn drain_and_shutdown(&self) {
1250        // WASM: Use unsafe cast to get mutable access
1251        let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1252        if let Err(e) = self_mut.sync_now() {
1253            log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1254        }
1255        *lock_mutex!(self.auto_sync_interval) = None;
1256    }
1257
1258    #[cfg(not(target_arch = "wasm32"))]
1259    pub fn drain_and_shutdown(&mut self) {
1260        if let Err(e) = self.sync_now() {
1261            log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1262        }
1263        *lock_mutex!(self.auto_sync_interval) = None;
1264        #[cfg(not(target_arch = "wasm32"))]
1265        {
1266            if let Some(stop) = &self.auto_sync_stop {
1267                stop.store(true, Ordering::SeqCst);
1268            }
1269            if let Some(handle) = self.auto_sync_thread.take() {
1270                let _ = handle.join();
1271            }
1272            if let Some(handle) = self.debounce_thread.take() {
1273                let _ = handle.join();
1274            }
1275            if let Some(task) = self.tokio_timer_task.take() {
1276                task.abort();
1277            }
1278            if let Some(task) = self.tokio_debounce_task.take() {
1279                task.abort();
1280            }
1281            self.auto_sync_stop = None;
1282            self.threshold_hit.store(false, Ordering::SeqCst);
1283        }
1284    }
1285
1286    pub fn clear_cache(&self) {
1287        lock_mutex!(self.cache).clear();
1288        lock_mutex!(self.lru_order).clear();
1289    }
1290
1291    /// Handle notification that the database has been imported
1292    ///
1293    /// This method should be called after a database import to ensure
1294    /// that any cached data is invalidated and fresh data is read from storage.
1295    ///
1296    /// # Returns
1297    /// * `Ok(())` - Cache cleared successfully
1298    /// * `Err(DatabaseError)` - If cache clearing fails
1299    ///
1300    /// # Example
1301    /// ```rust,no_run
1302    /// # use absurder_sql::storage::block_storage::BlockStorage;
1303    /// # async fn example() -> Result<(), absurder_sql::types::DatabaseError> {
1304    /// let mut storage = BlockStorage::new("mydb").await?;
1305    /// // ... database is imported externally ...
1306    /// storage.on_database_import().await?;
1307    /// # Ok(())
1308    /// # }
1309    /// ```
1310    pub async fn on_database_import(&mut self) -> Result<(), DatabaseError> {
1311        log::info!(
1312            "Clearing cache for database '{}' after import",
1313            self.db_name
1314        );
1315
1316        // Clear the LRU cache to force re-reading from storage
1317        self.clear_cache();
1318
1319        // Also clear dirty blocks since they're now stale
1320        lock_mutex!(self.dirty_blocks).clear();
1321
1322        // Clear checksum manager's cache to reload from new metadata
1323        self.checksum_manager.clear_checksums();
1324
1325        // Reload allocated blocks from global storage/allocation map
1326        #[cfg(target_arch = "wasm32")]
1327        {
1328            use super::vfs_sync::with_global_allocation_map;
1329            *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1330                gam.borrow()
1331                    .get(&self.db_name)
1332                    .cloned()
1333                    .unwrap_or_else(std::collections::HashSet::new)
1334            });
1335            log::debug!(
1336                "Reloaded {} allocated blocks from global allocation map",
1337                lock_mutex!(self.allocated_blocks).len()
1338            );
1339
1340            // Checksums are now managed by ChecksumManager, which loads from metadata on demand
1341            log::debug!("Checksum data will be reloaded from metadata on next verification");
1342        }
1343
1344        #[cfg(not(target_arch = "wasm32"))]
1345        {
1346            #[cfg(feature = "fs_persist")]
1347            {
1348                // Reload from filesystem allocations.json
1349                let mut alloc_path = self.base_dir.clone();
1350                alloc_path.push(&self.db_name);
1351                alloc_path.push("allocations.json");
1352
1353                if let Ok(content) = std::fs::read_to_string(&alloc_path) {
1354                    if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
1355                        if let Some(allocated_array) = alloc_data["allocated"].as_array() {
1356                            lock_mutex!(self.allocated_blocks).clear();
1357                            for block_id_val in allocated_array {
1358                                if let Some(block_id) = block_id_val.as_u64() {
1359                                    lock_mutex!(self.allocated_blocks).insert(block_id);
1360                                }
1361                            }
1362                            log::debug!(
1363                                "Reloaded {} allocated blocks from filesystem",
1364                                lock_mutex!(self.allocated_blocks).len()
1365                            );
1366                        }
1367                    }
1368                }
1369
1370                // Reload checksums from filesystem metadata.json
1371                let mut meta_path = self.base_dir.clone();
1372                meta_path.push(&self.db_name);
1373                meta_path.push("metadata.json");
1374
1375                if let Ok(content) = std::fs::read_to_string(&meta_path) {
1376                    if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
1377                        if let Some(entries) = meta_data["entries"].as_array() {
1378                            let mut new_checksums = HashMap::new();
1379                            let mut new_algos = HashMap::new();
1380
1381                            for entry in entries {
1382                                if let (Some(block_id), Some(checksum), Some(algo_str)) = (
1383                                    entry[0].as_u64(),
1384                                    entry[1]["checksum"].as_u64(),
1385                                    entry[1]["algo"].as_str(),
1386                                ) {
1387                                    new_checksums.insert(block_id, checksum);
1388
1389                                    let algo = match algo_str {
1390                                        "CRC32" => super::metadata::ChecksumAlgorithm::CRC32,
1391                                        _ => super::metadata::ChecksumAlgorithm::FastHash,
1392                                    };
1393                                    new_algos.insert(block_id, algo);
1394                                }
1395                            }
1396
1397                            self.checksum_manager
1398                                .replace_all(new_checksums.clone(), new_algos);
1399                            log::debug!(
1400                                "Reloaded {} checksums from filesystem metadata",
1401                                new_checksums.len()
1402                            );
1403                        }
1404                    }
1405                } else {
1406                    log::debug!("No metadata file found, checksums will be empty after import");
1407                }
1408            }
1409
1410            #[cfg(not(feature = "fs_persist"))]
1411            {
1412                // Native test mode: reload from GLOBAL_ALLOCATION_MAP
1413                use super::vfs_sync::with_global_allocation_map;
1414
1415                *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1416                    #[cfg(target_arch = "wasm32")]
1417                    let map = gam;
1418                    #[cfg(not(target_arch = "wasm32"))]
1419                    let map = gam.borrow();
1420                    map.get(&self.db_name)
1421                        .cloned()
1422                        .unwrap_or_else(std::collections::HashSet::new)
1423                });
1424                log::debug!(
1425                    "Reloaded {} allocated blocks from global allocation map (native test)",
1426                    lock_mutex!(self.allocated_blocks).len()
1427                );
1428
1429                // Checksums are managed by ChecksumManager, loaded from metadata on demand
1430                log::debug!("Checksum data will be reloaded from metadata on next verification");
1431            }
1432        }
1433
1434        log::info!(
1435            "Cache and allocation state refreshed for '{}'",
1436            self.db_name
1437        );
1438
1439        Ok(())
1440    }
1441
1442    /// Reload cache from GLOBAL_STORAGE (WASM only, for multi-connection support)
1443    #[cfg(target_arch = "wasm32")]
1444    pub fn reload_cache_from_global_storage(&self) {
1445        use crate::storage::vfs_sync::{with_global_metadata, with_global_storage};
1446
1447        log::info!("[RELOAD] Starting cache reload for: {}", self.db_name);
1448
1449        #[cfg(target_arch = "wasm32")]
1450        web_sys::console::log_1(
1451            &format!("[RELOAD] BlockStorage.db_name = {}", self.db_name).into(),
1452        );
1453
1454        let fresh_cache = with_global_storage(|storage_map| {
1455            if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
1456                log::info!(
1457                    "[RELOAD] Found {} blocks in GLOBAL_STORAGE",
1458                    db_storage.len()
1459                );
1460                db_storage.clone()
1461            } else {
1462                log::warn!(
1463                    "[RELOAD] No blocks found in GLOBAL_STORAGE for {}",
1464                    self.db_name
1465                );
1466                std::collections::HashMap::new()
1467            }
1468        });
1469
1470        let block_count = fresh_cache.len();
1471        log::info!(
1472            "[RELOAD] Loading {} blocks into cache and marking as dirty",
1473            block_count
1474        );
1475
1476        // CRITICAL: Also reload checksums from GLOBAL_METADATA to match the fresh cache
1477        // Without this, cached reads will verify against stale checksums (e.g., after import)
1478        with_global_metadata(|metadata_map| {
1479            if let Some(db_metadata) = metadata_map.borrow().get(&self.db_name) {
1480                log::info!("[RELOAD] Found {} metadata entries", db_metadata.len());
1481                // Clear existing checksums
1482                self.checksum_manager.clear_checksums();
1483
1484                // Load checksums from metadata
1485                let mut new_checksums = std::collections::HashMap::new();
1486                let mut new_algos = std::collections::HashMap::new();
1487                for (block_id, meta) in db_metadata {
1488                    new_checksums.insert(*block_id, meta.checksum);
1489                    new_algos.insert(*block_id, meta.algo);
1490                }
1491                self.checksum_manager.replace_all(new_checksums, new_algos);
1492            } else {
1493                log::warn!("[RELOAD] No metadata found for {}", self.db_name);
1494                // No metadata exists, clear checksums
1495                self.checksum_manager.clear_checksums();
1496            }
1497        });
1498
1499        // Replace cache contents while preserving LRU order for blocks that still exist
1500        let old_lru = {
1501            let mut lru = lock_mutex!(self.lru_order);
1502            std::mem::replace(&mut *lru, std::collections::VecDeque::new())
1503        };
1504        lock_mutex!(self.cache).clear();
1505
1506        // Insert new cache data (no need to mark dirty - sync reads from GLOBAL_STORAGE)
1507        for (block_id, block_data) in fresh_cache {
1508            lock_mutex!(self.cache).insert(block_id, block_data);
1509        }
1510
1511        log::info!(
1512            "[RELOAD] Cache reloaded with {} blocks",
1513            lock_mutex!(self.cache).len()
1514        );
1515
1516        // Restore LRU order for blocks that still exist, then add new blocks
1517        for block_id in old_lru {
1518            if lock_mutex!(self.cache).contains_key(&block_id) {
1519                lock_mutex!(self.lru_order).push_back(block_id);
1520            }
1521        }
1522
1523        // Add any new blocks not in the old LRU order
1524        let block_ids: Vec<u64> = lock_mutex!(self.cache).keys().copied().collect();
1525        for block_id in block_ids {
1526            if !lock_mutex!(self.lru_order).contains(&block_id) {
1527                lock_mutex!(self.lru_order).push_back(block_id);
1528            }
1529        }
1530
1531        log::info!(
1532            "[RELOAD] Reload complete: {} blocks in cache, {} dirty",
1533            lock_mutex!(self.cache).len(),
1534            lock_mutex!(self.dirty_blocks).len()
1535        );
1536
1537        // CRITICAL DEBUG: Check if block 0 is valid SQLite header
1538        #[cfg(target_arch = "wasm32")]
1539        {
1540            if let Some(block_0) = lock_mutex!(self.cache).get(&0) {
1541                let is_valid = block_0.len() >= 16 && &block_0[0..16] == b"SQLite format 3\0";
1542                web_sys::console::log_1(
1543                    &format!(
1544                        "[RELOAD] Block 0 in cache: {} bytes, valid SQLite header: {}",
1545                        block_0.len(),
1546                        is_valid
1547                    )
1548                    .into(),
1549                );
1550                if block_0.len() >= 16 {
1551                    web_sys::console::log_1(
1552                        &format!("[RELOAD] Block 0 header [0..16]: {:02x?}", &block_0[0..16])
1553                            .into(),
1554                    );
1555                }
1556
1557                // CRITICAL: Check database size field at offset 24-27
1558                if block_0.len() >= 28 {
1559                    let db_size_bytes = [block_0[24], block_0[25], block_0[26], block_0[27]];
1560                    let db_size_pages = u32::from_be_bytes(db_size_bytes);
1561                    web_sys::console::log_1(
1562                        &format!(
1563                            "[RELOAD] Database size field (offset 24-27): {} pages ({:02x?})",
1564                            db_size_pages, db_size_bytes
1565                        )
1566                        .into(),
1567                    );
1568                }
1569
1570                // DUMP FULL HEADER for analysis
1571                if block_0.len() >= 100 {
1572                    web_sys::console::log_1(
1573                        &format!(
1574                            "[RELOAD] Full header dump [16-31]: {:02x?}",
1575                            &block_0[16..32]
1576                        )
1577                        .into(),
1578                    );
1579                    web_sys::console::log_1(
1580                        &format!(
1581                            "[RELOAD] Full header dump [32-47]: {:02x?}",
1582                            &block_0[32..48]
1583                        )
1584                        .into(),
1585                    );
1586                    web_sys::console::log_1(
1587                        &format!(
1588                            "[RELOAD] Full header dump [48-63]: {:02x?}",
1589                            &block_0[48..64]
1590                        )
1591                        .into(),
1592                    );
1593                    web_sys::console::log_1(
1594                        &format!(
1595                            "[RELOAD] Full header dump [64-79]: {:02x?}",
1596                            &block_0[64..80]
1597                        )
1598                        .into(),
1599                    );
1600                    web_sys::console::log_1(
1601                        &format!(
1602                            "[RELOAD] Full header dump [80-99]: {:02x?}",
1603                            &block_0[80..100]
1604                        )
1605                        .into(),
1606                    );
1607                }
1608            } else {
1609                web_sys::console::log_1(
1610                    &format!("[RELOAD] ERROR: Block 0 NOT in cache after reload!").into(),
1611                );
1612            }
1613        }
1614    }
1615
1616    pub fn get_cache_size(&self) -> usize {
1617        lock_mutex!(self.cache).len()
1618    }
1619
1620    pub fn get_dirty_count(&self) -> usize {
1621        lock_mutex!(self.dirty_blocks).len()
1622    }
1623
1624    pub fn get_db_name(&self) -> &str {
1625        &self.db_name
1626    }
1627
1628    pub fn is_cached(&self, block_id: u64) -> bool {
1629        lock_mutex!(self.cache).contains_key(&block_id)
1630    }
1631
1632    /// Allocate a new block and return its ID
1633    pub async fn allocate_block(&mut self) -> Result<u64, DatabaseError> {
1634        super::allocation::allocate_block_impl(self).await
1635    }
1636
1637    /// Deallocate a block and mark it as available for reuse
1638    pub async fn deallocate_block(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1639        super::allocation::deallocate_block_impl(self, block_id).await
1640    }
1641
1642    /// Get the number of currently allocated blocks
1643    pub fn get_allocated_count(&self) -> usize {
1644        lock_mutex!(self.allocated_blocks).len()
1645    }
1646
1647    /// Crash simulation: simulate crash during IndexedDB commit
1648    /// If `blocks_written` is true, blocks are written to IndexedDB but commit marker doesn't advance
1649    /// If `blocks_written` is false, crash occurs before blocks are written
1650    #[cfg(target_arch = "wasm32")]
1651    pub async fn crash_simulation_sync(
1652        &mut self,
1653        blocks_written: bool,
1654    ) -> Result<(), DatabaseError> {
1655        log::info!(
1656            "CRASH SIMULATION: Starting crash simulation with blocks_written={}",
1657            blocks_written
1658        );
1659
1660        if blocks_written {
1661            // Simulate crash after blocks are written but before commit marker advances
1662            // This is the most critical crash scenario to test
1663
1664            // Step 1: Write blocks to IndexedDB (simulate partial transaction completion)
1665            let dirty_blocks = {
1666                let dirty = lock_mutex!(self.dirty_blocks);
1667                dirty.clone()
1668            };
1669
1670            if !dirty_blocks.is_empty() {
1671                log::info!(
1672                    "CRASH SIMULATION: Writing {} blocks to IndexedDB before crash",
1673                    dirty_blocks.len()
1674                );
1675
1676                // Use the existing IndexedDB persistence logic but don't advance commit marker
1677                let next_commit = self.get_commit_marker() + 1;
1678                let metadata_to_persist: Vec<(u64, BlockMetadataPersist)> = dirty_blocks
1679                    .iter()
1680                    .map(|(&block_id, block_data)| {
1681                        let checksum =
1682                            self.checksum_manager
1683                                .get_checksum(block_id)
1684                                .unwrap_or_else(|| {
1685                                    ChecksumManager::compute_checksum_with(
1686                                        block_data,
1687                                        self.checksum_manager.get_algorithm(block_id),
1688                                    )
1689                                });
1690                        (
1691                            block_id,
1692                            BlockMetadataPersist {
1693                                checksum,
1694                                last_modified_ms: BlockStorage::now_millis(),
1695                                version: next_commit as u32,
1696                                algo: self.checksum_manager.get_algorithm(block_id),
1697                            },
1698                        )
1699                    })
1700                    .collect();
1701
1702                log::debug!(
1703                    "CRASH SIMULATION: About to call persist_to_indexeddb for {} blocks",
1704                    dirty_blocks.len()
1705                );
1706
1707                // Write blocks and metadata to IndexedDB
1708                super::wasm_indexeddb::persist_to_indexeddb(
1709                    &self.db_name,
1710                    dirty_blocks,
1711                    metadata_to_persist,
1712                )
1713                .await?;
1714
1715                log::info!("CRASH SIMULATION: persist_to_indexeddb completed successfully");
1716                log::info!(
1717                    "CRASH SIMULATION: Blocks written to IndexedDB, simulating crash before commit marker advance"
1718                );
1719
1720                // Clear dirty blocks (they're now in IndexedDB)
1721                lock_mutex!(self.dirty_blocks).clear();
1722
1723                // DON'T advance commit marker - this simulates the crash
1724                // In a real crash, the commit marker update would fail
1725
1726                return Ok(());
1727            } else {
1728                log::info!("CRASH SIMULATION: No dirty blocks to write");
1729                return Ok(());
1730            }
1731        } else {
1732            // Simulate crash before blocks are written
1733            log::info!("CRASH SIMULATION: Simulating crash before blocks are written to IndexedDB");
1734
1735            // Just return success - blocks remain dirty, nothing written to IndexedDB
1736            return Ok(());
1737        }
1738    }
1739
1740    /// Crash simulation: simulate partial block writes during IndexedDB commit
1741    /// Only specified blocks are written to IndexedDB before crash
1742    #[cfg(target_arch = "wasm32")]
1743    pub async fn crash_simulation_partial_sync(
1744        &mut self,
1745        blocks_to_write: &[u64],
1746    ) -> Result<(), DatabaseError> {
1747        log::info!(
1748            "CRASH SIMULATION: Starting partial crash simulation for {} blocks",
1749            blocks_to_write.len()
1750        );
1751
1752        let dirty_blocks = {
1753            let dirty = lock_mutex!(self.dirty_blocks);
1754            dirty.clone()
1755        };
1756
1757        // Filter to only the blocks we want to "successfully" write before crash
1758        let partial_blocks: std::collections::HashMap<u64, Vec<u8>> = dirty_blocks
1759            .into_iter()
1760            .filter(|(block_id, _)| blocks_to_write.contains(block_id))
1761            .collect();
1762
1763        if !partial_blocks.is_empty() {
1764            log::info!(
1765                "CRASH SIMULATION: Writing {} out of {} blocks before crash",
1766                partial_blocks.len(),
1767                blocks_to_write.len()
1768            );
1769
1770            let next_commit = self.get_commit_marker() + 1;
1771            let metadata_to_persist: Vec<(u64, BlockMetadataPersist)> = partial_blocks
1772                .iter()
1773                .map(|(&block_id, block_data)| {
1774                    let checksum =
1775                        self.checksum_manager
1776                            .get_checksum(block_id)
1777                            .unwrap_or_else(|| {
1778                                ChecksumManager::compute_checksum_with(
1779                                    block_data,
1780                                    self.checksum_manager.get_algorithm(block_id),
1781                                )
1782                            });
1783                    (
1784                        block_id,
1785                        BlockMetadataPersist {
1786                            checksum,
1787                            last_modified_ms: BlockStorage::now_millis(),
1788                            version: next_commit as u32,
1789                            algo: self.checksum_manager.get_algorithm(block_id),
1790                        },
1791                    )
1792                })
1793                .collect();
1794
1795            // Write only the partial blocks to IndexedDB
1796            super::wasm_indexeddb::persist_to_indexeddb(
1797                &self.db_name,
1798                partial_blocks.clone(),
1799                metadata_to_persist,
1800            )
1801            .await?;
1802
1803            // Remove only the written blocks from dirty_blocks
1804            {
1805                let mut dirty = lock_mutex!(self.dirty_blocks);
1806                for block_id in partial_blocks.keys() {
1807                    dirty.remove(block_id);
1808                }
1809            }
1810
1811            log::info!(
1812                "CRASH SIMULATION: Partial blocks written, simulating crash before commit marker advance"
1813            );
1814
1815            // DON'T advance commit marker - simulates crash during transaction
1816        }
1817
1818        Ok(())
1819    }
1820
1821    /// Perform crash recovery: detect and handle incomplete IndexedDB transactions
1822    /// This method detects inconsistencies between IndexedDB state and commit markers
1823    /// and either finalizes or rolls back incomplete transactions
1824    #[cfg(target_arch = "wasm32")]
1825    pub async fn perform_crash_recovery(&mut self) -> Result<CrashRecoveryAction, DatabaseError> {
1826        log::info!(
1827            "CRASH RECOVERY: Starting crash recovery scan for database: {}",
1828            self.db_name
1829        );
1830
1831        // Step 1: Get current commit marker
1832        let current_marker = self.get_commit_marker();
1833        log::info!("CRASH RECOVERY: Current commit marker: {}", current_marker);
1834
1835        // Step 2: Scan IndexedDB for blocks with versions > commit marker
1836        // These represent incomplete transactions that need recovery
1837        let inconsistent_blocks = self.scan_for_inconsistent_blocks(current_marker).await?;
1838
1839        if inconsistent_blocks.is_empty() {
1840            log::info!("CRASH RECOVERY: No inconsistent blocks found, system is consistent");
1841            return Ok(CrashRecoveryAction::NoActionNeeded);
1842        }
1843
1844        log::info!(
1845            "CRASH RECOVERY: Found {} inconsistent blocks that need recovery",
1846            inconsistent_blocks.len()
1847        );
1848
1849        // Step 3: Determine recovery action based on transaction completeness
1850        let recovery_action = self.determine_recovery_action(&inconsistent_blocks).await?;
1851
1852        match recovery_action {
1853            CrashRecoveryAction::Rollback => {
1854                log::info!("CRASH RECOVERY: Performing rollback of incomplete transaction");
1855                self.rollback_incomplete_transaction(&inconsistent_blocks)
1856                    .await?;
1857            }
1858            CrashRecoveryAction::Finalize => {
1859                log::info!("CRASH RECOVERY: Performing finalization of complete transaction");
1860                self.finalize_complete_transaction(&inconsistent_blocks)
1861                    .await?;
1862            }
1863            CrashRecoveryAction::NoActionNeeded => {
1864                // Already handled above
1865            }
1866        }
1867
1868        log::info!("CRASH RECOVERY: Recovery completed successfully");
1869        Ok(recovery_action)
1870    }
1871
1872    /// Scan IndexedDB for blocks with versions greater than the commit marker
1873    #[cfg(target_arch = "wasm32")]
1874    async fn scan_for_inconsistent_blocks(
1875        &self,
1876        commit_marker: u64,
1877    ) -> Result<Vec<(u64, u64)>, DatabaseError> {
1878        log::info!(
1879            "CRASH RECOVERY: Scanning for blocks with version > {}",
1880            commit_marker
1881        );
1882
1883        // This is a simplified implementation - in a real system we'd scan IndexedDB directly
1884        // For now, we'll check the global metadata storage
1885        let mut inconsistent_blocks = Vec::new();
1886
1887        vfs_sync::with_global_metadata(|meta_map| {
1888            if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1889                for (block_id, metadata) in db_meta.iter() {
1890                    if metadata.version as u64 > commit_marker {
1891                        log::info!(
1892                            "CRASH RECOVERY: Found inconsistent block {} with version {} > marker {}",
1893                            block_id,
1894                            metadata.version,
1895                            commit_marker
1896                        );
1897                        inconsistent_blocks.push((*block_id, metadata.version as u64));
1898                    }
1899                }
1900            }
1901        });
1902
1903        Ok(inconsistent_blocks)
1904    }
1905
1906    /// Determine whether to rollback or finalize based on transaction completeness
1907    #[cfg(target_arch = "wasm32")]
1908    async fn determine_recovery_action(
1909        &self,
1910        inconsistent_blocks: &[(u64, u64)],
1911    ) -> Result<CrashRecoveryAction, DatabaseError> {
1912        // Simple heuristic: if all inconsistent blocks have the same version (next expected commit),
1913        // then the transaction was likely complete and should be finalized.
1914        // Otherwise, rollback to maintain consistency.
1915
1916        let expected_next_commit = self.get_commit_marker() + 1;
1917        let all_same_version = inconsistent_blocks
1918            .iter()
1919            .all(|(_, version)| *version == expected_next_commit);
1920
1921        if all_same_version && !inconsistent_blocks.is_empty() {
1922            log::info!(
1923                "CRASH RECOVERY: All inconsistent blocks have expected version {}, finalizing transaction",
1924                expected_next_commit
1925            );
1926            Ok(CrashRecoveryAction::Finalize)
1927        } else {
1928            log::info!(
1929                "CRASH RECOVERY: Inconsistent block versions detected, rolling back transaction"
1930            );
1931            Ok(CrashRecoveryAction::Rollback)
1932        }
1933    }
1934
1935    /// Rollback incomplete transaction by removing inconsistent blocks
1936    #[cfg(target_arch = "wasm32")]
1937    async fn rollback_incomplete_transaction(
1938        &mut self,
1939        inconsistent_blocks: &[(u64, u64)],
1940    ) -> Result<(), DatabaseError> {
1941        log::info!(
1942            "CRASH RECOVERY: Rolling back {} inconsistent blocks",
1943            inconsistent_blocks.len()
1944        );
1945
1946        // Remove inconsistent blocks from global metadata
1947        vfs_sync::with_global_metadata(|meta_map| {
1948            if let Some(db_meta) = meta_map.borrow_mut().get_mut(&self.db_name) {
1949                for (block_id, _) in inconsistent_blocks {
1950                    log::info!(
1951                        "CRASH RECOVERY: Removing inconsistent block {} from metadata",
1952                        block_id
1953                    );
1954                    db_meta.remove(block_id);
1955                }
1956            }
1957        });
1958
1959        // Remove inconsistent blocks from global storage
1960        vfs_sync::with_global_storage(|storage_map| {
1961            if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
1962                for (block_id, _) in inconsistent_blocks {
1963                    log::info!(
1964                        "CRASH RECOVERY: Removing inconsistent block {} from global storage",
1965                        block_id
1966                    );
1967                    db_storage.remove(block_id);
1968                }
1969            }
1970        });
1971
1972        // Clear any cached data for these blocks
1973        for (block_id, _) in inconsistent_blocks {
1974            lock_mutex!(self.cache).remove(block_id);
1975            // Remove from LRU order
1976            lock_mutex!(self.lru_order).retain(|&id| id != *block_id);
1977        }
1978
1979        // Remove inconsistent blocks from IndexedDB to avoid accumulating orphaned data
1980        let block_ids_to_delete: Vec<u64> = inconsistent_blocks.iter().map(|(id, _)| *id).collect();
1981        if !block_ids_to_delete.is_empty() {
1982            log::info!(
1983                "CRASH RECOVERY: Deleting {} blocks from IndexedDB",
1984                block_ids_to_delete.len()
1985            );
1986            super::wasm_indexeddb::delete_blocks_from_indexeddb(
1987                &self.db_name,
1988                &block_ids_to_delete,
1989            )
1990            .await?;
1991            log::info!("CRASH RECOVERY: Successfully deleted blocks from IndexedDB");
1992        }
1993
1994        log::info!("CRASH RECOVERY: Rollback completed");
1995        Ok(())
1996    }
1997
1998    /// Finalize complete transaction by advancing commit marker
1999    #[cfg(target_arch = "wasm32")]
2000    async fn finalize_complete_transaction(
2001        &mut self,
2002        inconsistent_blocks: &[(u64, u64)],
2003    ) -> Result<(), DatabaseError> {
2004        log::info!(
2005            "CRASH RECOVERY: Finalizing transaction for {} blocks",
2006            inconsistent_blocks.len()
2007        );
2008
2009        // Find the target commit marker (should be consistent across all blocks)
2010        if let Some((_, target_version)) = inconsistent_blocks.first() {
2011            let new_commit_marker = *target_version;
2012
2013            // Advance the commit marker to make the blocks visible
2014            vfs_sync::with_global_commit_marker(|cm| {
2015                cm.borrow_mut()
2016                    .insert(self.db_name.clone(), new_commit_marker);
2017            });
2018
2019            log::info!(
2020                "CRASH RECOVERY: Advanced commit marker from {} to {}",
2021                self.get_commit_marker(),
2022                new_commit_marker
2023            );
2024
2025            // Update checksums for the finalized blocks
2026            for (block_id, _) in inconsistent_blocks {
2027                // Read the block data to compute and store checksum
2028                if let Ok(data) = self.read_block_sync(*block_id) {
2029                    self.checksum_manager.store_checksum(*block_id, &data);
2030                    log::info!(
2031                        "CRASH RECOVERY: Updated checksum for finalized block {}",
2032                        block_id
2033                    );
2034                }
2035            }
2036        }
2037
2038        log::info!("CRASH RECOVERY: Finalization completed");
2039        Ok(())
2040    }
2041
2042    // Leader Election Methods (WASM only)
2043
2044    /// Start leader election process
2045    #[cfg(target_arch = "wasm32")]
2046    pub async fn start_leader_election(&self) -> Result<(), DatabaseError> {
2047        if self.leader_election.borrow().is_none() {
2048            log::debug!(
2049                "BlockStorage::start_leader_election() - Creating new LeaderElectionManager for {}",
2050                self.db_name
2051            );
2052            let mut manager =
2053                super::leader_election::LeaderElectionManager::new(self.db_name.clone());
2054            log::debug!("BlockStorage::start_leader_election() - Calling manager.start_election()");
2055            manager.start_election().await?;
2056            log::debug!(
2057                "BlockStorage::start_leader_election() - Election started, storing manager"
2058            );
2059            *self.leader_election.borrow_mut() = Some(manager);
2060        } else {
2061            // If election is already running, force leadership takeover (requestLeadership)
2062            log::debug!(
2063                "BlockStorage::start_leader_election() - Election already exists, forcing leadership"
2064            );
2065            if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2066                manager.force_become_leader().await?;
2067            }
2068        }
2069        Ok(())
2070    }
2071
2072    /// Check if this instance WAS the leader WITHOUT triggering re-election
2073    /// Used during cleanup to avoid starting new elections
2074    #[cfg(target_arch = "wasm32")]
2075    pub fn was_leader(&self) -> bool {
2076        self.leader_election
2077            .borrow()
2078            .as_ref()
2079            .map(|m| m.state.borrow().is_leader)
2080            .unwrap_or(false)
2081    }
2082
2083    /// Stop heartbeat interval synchronously (for Drop implementation)
2084    /// This is idempotent and safe to call multiple times
2085    /// Silently skips if already stopped or if manager is borrowed
2086    #[cfg(target_arch = "wasm32")]
2087    pub fn stop_heartbeat_sync(&self) {
2088        // Use try_borrow_mut to avoid deadlock when multiple DBs drop simultaneously
2089        if let Ok(mut manager_ref) = self.leader_election.try_borrow_mut() {
2090            if let Some(ref mut manager) = *manager_ref {
2091                // Only clear if interval exists (idempotent)
2092                if let Some(interval_id) = manager.heartbeat_interval.take() {
2093                    if let Some(window) = web_sys::window() {
2094                        window.clear_interval_with_handle(interval_id);
2095                        web_sys::console::log_1(
2096                            &format!(
2097                                "[DROP] Cleared heartbeat interval {} for {}",
2098                                interval_id, self.db_name
2099                            )
2100                            .into(),
2101                        );
2102                    }
2103                }
2104                // Note: heartbeat closure is intentionally leaked (via forget())
2105                // and becomes a no-op when heartbeat_valid is set to false
2106            }
2107        } else {
2108            // Already borrowed (e.g., first DB is still dropping) - skip silently
2109            web_sys::console::log_1(
2110                &format!(
2111                    "[DROP] Skipping heartbeat stop for {} (already handled)",
2112                    self.db_name
2113                )
2114                .into(),
2115            );
2116        }
2117    }
2118
2119    /// Check if this instance is the leader (with re-election on lease expiry)
2120    #[cfg(target_arch = "wasm32")]
2121    pub async fn is_leader(&self) -> bool {
2122        // Start leader election if not already started
2123        if self.leader_election.borrow().is_none() {
2124            log::debug!(
2125                "BlockStorage::is_leader() - Starting leader election for {}",
2126                self.db_name
2127            );
2128            if let Err(e) = self.start_leader_election().await {
2129                log::error!("Failed to start leader election: {:?}", e);
2130                return false;
2131            }
2132            log::debug!("BlockStorage::is_leader() - Leader election started successfully");
2133        } else {
2134            log::debug!(
2135                "BlockStorage::is_leader() - Leader election already exists for {}",
2136                self.db_name
2137            );
2138        }
2139
2140        if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2141            let is_leader = manager.is_leader().await;
2142
2143            // If no current leader (lease expired), trigger re-election
2144            if !is_leader {
2145                let state = manager.state.borrow();
2146                if state.leader_id.is_none() {
2147                    log::debug!(
2148                        "No current leader for {} - triggering re-election",
2149                        self.db_name
2150                    );
2151                    drop(state);
2152                    let _ = manager.try_become_leader().await;
2153
2154                    // Start heartbeat if we became leader
2155                    let new_is_leader = manager.state.borrow().is_leader;
2156                    if new_is_leader && manager.heartbeat_interval.is_none() {
2157                        let _ = manager.start_heartbeat();
2158                    }
2159
2160                    log::debug!(
2161                        "is_leader() for {} = {} (after re-election)",
2162                        self.db_name,
2163                        new_is_leader
2164                    );
2165                    return new_is_leader;
2166                }
2167            }
2168
2169            log::debug!("is_leader() for {} = {}", self.db_name, is_leader);
2170            is_leader
2171        } else {
2172            log::debug!("No leader election manager for {}", self.db_name);
2173            false
2174        }
2175    }
2176
2177    /// Stop leader election (e.g., when tab is closing)
2178    #[cfg(target_arch = "wasm32")]
2179    pub async fn stop_leader_election(&self) -> Result<(), DatabaseError> {
2180        if let Some(mut manager) = self
2181            .leader_election
2182            .try_borrow_mut()
2183            .expect("Failed to borrow leader_election in stop_leader_election")
2184            .take()
2185        {
2186            manager.stop_election().await?;
2187        }
2188        Ok(())
2189    }
2190
2191    /// Send a leader heartbeat (for testing)
2192    #[cfg(target_arch = "wasm32")]
2193    pub async fn send_leader_heartbeat(&self) -> Result<(), DatabaseError> {
2194        if let Some(ref manager) = *self.leader_election.borrow() {
2195            manager.send_heartbeat().await
2196        } else {
2197            Err(DatabaseError::new(
2198                "LEADER_ELECTION_ERROR",
2199                "Leader election not started",
2200            ))
2201        }
2202    }
2203
2204    /// Get timestamp of last received leader heartbeat
2205    #[cfg(target_arch = "wasm32")]
2206    pub async fn get_last_leader_heartbeat(&self) -> Result<u64, DatabaseError> {
2207        if let Some(ref manager) = *self.leader_election.borrow() {
2208            Ok(manager.get_last_heartbeat().await)
2209        } else {
2210            Err(DatabaseError::new(
2211                "LEADER_ELECTION_ERROR",
2212                "Leader election not started",
2213            ))
2214        }
2215    }
2216
2217    // Observability Methods
2218
2219    /// Get comprehensive metrics for observability
2220    pub fn get_metrics(&self) -> super::observability::StorageMetrics {
2221        let dirty_count = self.get_dirty_count();
2222        let dirty_bytes = dirty_count * BLOCK_SIZE;
2223
2224        #[cfg(not(target_arch = "wasm32"))]
2225        let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2226            (
2227                self.sync_count.load(Ordering::SeqCst),
2228                self.timer_sync_count.load(Ordering::SeqCst),
2229                self.debounce_sync_count.load(Ordering::SeqCst),
2230                self.last_sync_duration_ms.load(Ordering::SeqCst),
2231            )
2232        };
2233
2234        #[cfg(target_arch = "wasm32")]
2235        let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2236            // For WASM, use observability manager for sync_count tracking
2237            (self.observability.get_sync_count(), 0, 0, 1)
2238        };
2239
2240        let error_count = self.observability.get_error_count();
2241        let checksum_failures = self.observability.get_checksum_failures();
2242
2243        // Calculate throughput and error rate
2244        let total_operations = sync_count + error_count;
2245        let (throughput_blocks_per_sec, throughput_bytes_per_sec) = self
2246            .observability
2247            .calculate_throughput(last_sync_duration_ms);
2248        let error_rate = self.observability.calculate_error_rate(total_operations);
2249
2250        super::observability::StorageMetrics {
2251            dirty_count,
2252            dirty_bytes,
2253            sync_count,
2254            timer_sync_count,
2255            debounce_sync_count,
2256            error_count,
2257            checksum_failures,
2258            last_sync_duration_ms,
2259            throughput_blocks_per_sec,
2260            throughput_bytes_per_sec,
2261            error_rate,
2262        }
2263    }
2264
2265    /// Set sync event callbacks
2266    #[cfg(not(target_arch = "wasm32"))]
2267    pub fn set_sync_callbacks(
2268        &mut self,
2269        on_sync_start: super::observability::SyncStartCallback,
2270        on_sync_success: super::observability::SyncSuccessCallback,
2271        on_sync_failure: super::observability::SyncFailureCallback,
2272    ) {
2273        self.observability.sync_start_callback = Some(on_sync_start);
2274        self.observability.sync_success_callback = Some(on_sync_success);
2275        self.observability.sync_failure_callback = Some(on_sync_failure);
2276    }
2277
2278    /// Set backpressure callback
2279    #[cfg(not(target_arch = "wasm32"))]
2280    pub fn set_backpressure_callback(
2281        &mut self,
2282        callback: super::observability::BackpressureCallback,
2283    ) {
2284        self.observability.backpressure_callback = Some(callback);
2285    }
2286
2287    /// Set error callback
2288    #[cfg(not(target_arch = "wasm32"))]
2289    pub fn set_error_callback(&mut self, callback: super::observability::ErrorCallback) {
2290        self.observability.error_callback = Some(callback);
2291    }
2292
2293    /// Set WASM sync success callback
2294    #[cfg(target_arch = "wasm32")]
2295    pub fn set_sync_success_callback(
2296        &mut self,
2297        callback: super::observability::WasmSyncSuccessCallback,
2298    ) {
2299        self.observability.wasm_sync_success_callback = Some(callback);
2300    }
2301
2302    /// Check if auto-sync is currently enabled
2303    pub fn is_auto_sync_enabled(&self) -> bool {
2304        lock_mutex!(self.auto_sync_interval).is_some()
2305    }
2306
2307    /// Get the current sync policy (if any)
2308    pub fn get_sync_policy(&self) -> Option<super::SyncPolicy> {
2309        lock_mutex!(self.policy).clone()
2310    }
2311
2312    /// Force synchronization with durability guarantees
2313    ///
2314    /// This method ensures that all dirty blocks are persisted to durable storage
2315    /// (IndexedDB in WASM, filesystem in native) and waits for the operation to complete.
2316    /// This is called by VFS xSync to provide SQLite's durability guarantees.
2317    pub async fn force_sync(&mut self) -> Result<(), DatabaseError> {
2318        log::info!("force_sync: Starting forced synchronization with durability guarantees");
2319
2320        let dirty_count = self.get_dirty_count();
2321        if dirty_count == 0 {
2322            log::debug!("force_sync: No dirty blocks to sync");
2323            return Ok(());
2324        }
2325
2326        log::info!(
2327            "force_sync: Syncing {} dirty blocks with durability guarantee",
2328            dirty_count
2329        );
2330
2331        // Just use the regular sync - it already waits for persistence in WASM
2332        self.sync().await?;
2333
2334        log::info!("force_sync: Successfully completed forced synchronization");
2335        Ok(())
2336    }
2337
2338    /// Set telemetry metrics (used for instrumentation)
2339    #[cfg(feature = "telemetry")]
2340    pub fn set_metrics(&mut self, metrics: Option<crate::telemetry::Metrics>) {
2341        self.metrics = metrics;
2342    }
2343
2344    /// Get telemetry metrics
2345    #[cfg(feature = "telemetry")]
2346    pub fn metrics(&self) -> Option<&crate::telemetry::Metrics> {
2347        self.metrics.as_ref()
2348    }
2349
2350    /// Create a test instance with minimal setup
2351    #[cfg(feature = "telemetry")]
2352    pub fn new_for_test() -> Self {
2353        Self {
2354            cache: Mutex::new(HashMap::new()),
2355            dirty_blocks: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2356            allocated_blocks: Mutex::new(HashSet::new()),
2357            deallocated_blocks: Mutex::new(HashSet::new()),
2358            next_block_id: AtomicU64::new(1),
2359            capacity: 128,
2360            lru_order: Mutex::new(VecDeque::new()),
2361            checksum_manager: crate::storage::metadata::ChecksumManager::new(
2362                crate::storage::metadata::ChecksumAlgorithm::FastHash,
2363            ),
2364            #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
2365            base_dir: std::path::PathBuf::from("/tmp/test"),
2366            db_name: "test.db".to_string(),
2367            backend: StorageBackend::IndexedDb,
2368            auto_sync_interval: Mutex::new(None),
2369            #[cfg(not(target_arch = "wasm32"))]
2370            last_auto_sync: std::time::Instant::now(),
2371            policy: Mutex::new(None),
2372            #[cfg(not(target_arch = "wasm32"))]
2373            auto_sync_stop: None,
2374            #[cfg(not(target_arch = "wasm32"))]
2375            auto_sync_thread: None,
2376            #[cfg(not(target_arch = "wasm32"))]
2377            debounce_thread: None,
2378            #[cfg(not(target_arch = "wasm32"))]
2379            tokio_timer_task: None,
2380            #[cfg(not(target_arch = "wasm32"))]
2381            tokio_debounce_task: None,
2382            #[cfg(not(target_arch = "wasm32"))]
2383            last_write_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2384            #[cfg(not(target_arch = "wasm32"))]
2385            threshold_hit: Arc::new(std::sync::atomic::AtomicBool::new(false)),
2386            #[cfg(not(target_arch = "wasm32"))]
2387            sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2388            #[cfg(not(target_arch = "wasm32"))]
2389            timer_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2390            #[cfg(not(target_arch = "wasm32"))]
2391            debounce_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2392            #[cfg(not(target_arch = "wasm32"))]
2393            last_sync_duration_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2394            #[cfg(not(target_arch = "wasm32"))]
2395            sync_sender: None,
2396            #[cfg(not(target_arch = "wasm32"))]
2397            sync_receiver: None,
2398            recovery_report: RecoveryReport::default(),
2399            #[cfg(target_arch = "wasm32")]
2400            leader_election: std::cell::RefCell::new(None),
2401            observability: super::observability::ObservabilityManager::new(),
2402            metrics: None,
2403        }
2404    }
2405}
2406
2407#[cfg(all(test, target_arch = "wasm32"))]
2408mod wasm_commit_marker_tests {
2409    use super::*;
2410    use wasm_bindgen_test::*;
2411
2412    wasm_bindgen_test_configure!(run_in_browser);
2413
2414    // Helper: set commit marker for a db name in WASM global
2415    fn set_commit_marker(db: &str, v: u64) {
2416        // Normalize name to match BlockStorage.db_name
2417        let db = crate::utils::normalize_db_name(db);
2418        super::vfs_sync::with_global_commit_marker(|cm| {
2419            cm.borrow_mut().insert(db, v);
2420        });
2421    }
2422
2423    // Helper: get commit marker for a db name in WASM global
2424    fn get_commit_marker(db: &str) -> u64 {
2425        // Normalize name to match BlockStorage.db_name
2426        let db = crate::utils::normalize_db_name(db);
2427        vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2428    }
2429
2430    #[wasm_bindgen_test]
2431    async fn gating_returns_zeroed_until_marker_catches_up_wasm() {
2432        let db = "cm_gating_wasm";
2433        let mut s = BlockStorage::new(db).await.expect("create storage");
2434
2435        // Write a block (starts at version 1, uncommitted)
2436        let bid = s.allocate_block().await.expect("alloc block");
2437        let data_v1 = vec![0x33u8; BLOCK_SIZE];
2438        s.write_block(bid, data_v1.clone()).await.expect("write v1");
2439
2440        // Before sync, commit marker is 0, block version is 1, so should be invisible
2441        s.clear_cache();
2442        let out0 = s.read_block(bid).await.expect("read before commit");
2443        assert_eq!(
2444            out0,
2445            vec![0u8; BLOCK_SIZE],
2446            "uncommitted data must read as zeroed"
2447        );
2448
2449        // After sync, commit marker advances to 1, block version is 1, so should be visible
2450        s.sync().await.expect("sync v1");
2451        s.clear_cache();
2452        let out1 = s.read_block(bid).await.expect("read after commit");
2453        assert_eq!(out1, data_v1, "committed data should be visible");
2454    }
2455
2456    #[wasm_bindgen_test]
2457    async fn invisible_blocks_skip_checksum_verification_wasm() {
2458        let db = "cm_checksum_skip_wasm";
2459        let mut s = BlockStorage::new(db).await.expect("create storage");
2460
2461        let bid = s.allocate_block().await.expect("alloc block");
2462        let data = vec![0x44u8; BLOCK_SIZE];
2463        s.write_block(bid, data).await.expect("write v1");
2464        s.sync().await.expect("sync v1"); // commit marker advances to 1, block version is 1
2465
2466        // Make the block invisible by moving commit marker back to 0
2467        set_commit_marker(db, 0);
2468
2469        // Corrupt the stored checksum; invisible reads must NOT verify checksum
2470        s.set_block_checksum_for_testing(bid, 1234567);
2471        s.clear_cache();
2472        let out = s
2473            .read_block(bid)
2474            .await
2475            .expect("read while invisible should not error");
2476        assert_eq!(
2477            out,
2478            vec![0u8; BLOCK_SIZE],
2479            "invisible block reads as zeroed"
2480        );
2481
2482        // Now make it visible again; checksum verification should trigger and fail
2483        set_commit_marker(db, 1);
2484        s.clear_cache();
2485        let err = s
2486            .read_block(bid)
2487            .await
2488            .expect_err("expected checksum mismatch once visible");
2489        assert_eq!(err.code, "CHECKSUM_MISMATCH");
2490    }
2491
2492    #[wasm_bindgen_test]
2493    async fn commit_marker_advances_and_versions_track_syncs_wasm() {
2494        let db = "cm_versions_wasm";
2495        let mut s = BlockStorage::new_with_capacity(db, 8)
2496            .await
2497            .expect("create storage");
2498
2499        let b1 = s.allocate_block().await.expect("alloc b1");
2500        let b2 = s.allocate_block().await.expect("alloc b2");
2501
2502        s.write_block(b1, vec![1u8; BLOCK_SIZE])
2503            .await
2504            .expect("write b1 v1");
2505        s.write_block(b2, vec![2u8; BLOCK_SIZE])
2506            .await
2507            .expect("write b2 v1");
2508        s.sync().await.expect("sync #1");
2509
2510        let cm1 = get_commit_marker(db);
2511        assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2512        let meta1 = s.get_block_metadata_for_testing();
2513        assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2514        assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2515
2516        // Update only b1 and sync again; only b1's version should bump
2517        s.write_block(b1, vec![3u8; BLOCK_SIZE])
2518            .await
2519            .expect("write b1 v2");
2520        s.sync().await.expect("sync #2");
2521
2522        let cm2 = get_commit_marker(db);
2523        assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2524        let meta2 = s.get_block_metadata_for_testing();
2525        assert_eq!(
2526            meta2.get(&b1).unwrap().1 as u64,
2527            cm2,
2528            "updated block tracks new version"
2529        );
2530        assert_eq!(
2531            meta2.get(&b2).unwrap().1 as u64,
2532            1,
2533            "unchanged block retains prior version"
2534        );
2535    }
2536}
2537
2538#[cfg(all(test, not(target_arch = "wasm32"), not(feature = "fs_persist")))]
2539mod commit_marker_tests {
2540    use super::*;
2541
2542    // Helper: set commit marker for a db name in test-global mirror
2543    fn set_commit_marker(db: &str, v: u64) {
2544        // Normalize name to match BlockStorage.db_name
2545        let db = crate::utils::normalize_db_name(db);
2546        super::vfs_sync::with_global_commit_marker(|cm| {
2547            cm.borrow_mut().insert(db, v);
2548        });
2549    }
2550
2551    // Helper: get commit marker for a db name in test-global mirror
2552    fn get_commit_marker(db: &str) -> u64 {
2553        // Normalize name to match BlockStorage.db_name
2554        let db = crate::utils::normalize_db_name(db);
2555        super::vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2556    }
2557
2558    #[tokio::test(flavor = "current_thread")]
2559    async fn gating_returns_zeroed_until_marker_catches_up() {
2560        let db = "cm_gating_basic";
2561        println!("DEBUG: Creating BlockStorage for {}", db);
2562        let mut s = BlockStorage::new(db).await.expect("create storage");
2563        println!("DEBUG: BlockStorage created successfully");
2564
2565        // Write a block (starts at version 1, uncommitted)
2566        let bid = s.allocate_block().await.expect("alloc block");
2567        println!("DEBUG: Allocated block {}", bid);
2568        let data_v1 = vec![0x11u8; BLOCK_SIZE];
2569        s.write_block(bid, data_v1.clone()).await.expect("write v1");
2570        println!("DEBUG: Wrote block {} with data", bid);
2571
2572        // Before sync, commit marker is 0, block version is 1, so should be invisible
2573        s.clear_cache();
2574        let out0 = s.read_block(bid).await.expect("read before commit");
2575        assert_eq!(
2576            out0,
2577            vec![0u8; BLOCK_SIZE],
2578            "uncommitted data must read as zeroed"
2579        );
2580        println!("DEBUG: Pre-sync read returned zeroed data as expected");
2581
2582        // After sync, commit marker advances to 1, block version is 1, so should be visible
2583        println!("DEBUG: About to call sync");
2584        s.sync().await.expect("sync v1");
2585        println!("DEBUG: Sync completed successfully");
2586
2587        // Debug: Check commit marker and metadata after sync
2588        let commit_marker = get_commit_marker(db);
2589        println!("DEBUG: Commit marker after sync: {}", commit_marker);
2590
2591        s.clear_cache();
2592        let out1 = s.read_block(bid).await.expect("read after commit");
2593
2594        // Debug: Print what we got vs what we expected
2595        println!("DEBUG: Expected data: {:?}", &data_v1[..8]);
2596        println!("DEBUG: Actual data: {:?}", &out1[..8]);
2597        println!(
2598            "DEBUG: Data lengths - expected: {}, actual: {}",
2599            data_v1.len(),
2600            out1.len()
2601        );
2602
2603        // Check if data matches without panicking
2604        let data_matches = out1 == data_v1;
2605        println!("DEBUG: Data matches: {}", data_matches);
2606
2607        if !data_matches {
2608            println!("DEBUG: Data mismatch detected - investigating further");
2609            // Check if it's all zeros (uncommitted)
2610            let is_all_zeros = out1.iter().all(|&b| b == 0);
2611            println!("DEBUG: Is all zeros: {}", is_all_zeros);
2612
2613            // Check metadata and commit marker state
2614            println!("DEBUG: Final commit marker: {}", get_commit_marker(db));
2615
2616            panic!("Data mismatch: expected committed data to be visible after sync");
2617        }
2618
2619        println!("DEBUG: Test passed - data is visible after commit");
2620    }
2621
2622    #[tokio::test(flavor = "current_thread")]
2623    async fn invisible_blocks_skip_checksum_verification() {
2624        let db = "cm_checksum_skip";
2625        let mut s = BlockStorage::new(db).await.expect("create storage");
2626
2627        let bid = s.allocate_block().await.expect("alloc block");
2628        let data = vec![0xAAu8; BLOCK_SIZE];
2629        s.write_block(bid, data.clone()).await.expect("write v1");
2630        s.sync().await.expect("sync v1"); // commit marker advances to 1, block version is 1
2631
2632        // Make the block invisible by moving commit marker back to 0
2633        set_commit_marker(db, 0);
2634
2635        // Corrupt the stored checksum; invisible reads must NOT verify checksum
2636        s.set_block_checksum_for_testing(bid, 1234567);
2637        s.clear_cache();
2638        let out = s
2639            .read_block(bid)
2640            .await
2641            .expect("read while invisible should not error");
2642        assert_eq!(
2643            out,
2644            vec![0u8; BLOCK_SIZE],
2645            "invisible block reads as zeroed"
2646        );
2647
2648        // Now make it visible again; checksum verification should trigger and fail
2649        set_commit_marker(db, 1);
2650        s.clear_cache();
2651        let err = s
2652            .read_block(bid)
2653            .await
2654            .expect_err("expected checksum mismatch once visible");
2655        assert_eq!(err.code, "CHECKSUM_MISMATCH");
2656    }
2657
2658    #[tokio::test(flavor = "current_thread")]
2659    async fn commit_marker_advances_and_versions_track_syncs() {
2660        let db = "cm_versions";
2661        let mut s = BlockStorage::new_with_capacity(db, 8)
2662            .await
2663            .expect("create storage");
2664
2665        let b1 = s.allocate_block().await.expect("alloc b1");
2666        let b2 = s.allocate_block().await.expect("alloc b2");
2667
2668        s.write_block(b1, vec![1u8; BLOCK_SIZE])
2669            .await
2670            .expect("write b1 v1");
2671        s.write_block(b2, vec![2u8; BLOCK_SIZE])
2672            .await
2673            .expect("write b2 v1");
2674        s.sync().await.expect("sync #1");
2675
2676        let cm1 = get_commit_marker(db);
2677        assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2678        let meta1 = s.get_block_metadata_for_testing();
2679        assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2680        assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2681
2682        // Update only b1 and sync again; only b1's version should bump
2683        s.write_block(b1, vec![3u8; BLOCK_SIZE])
2684            .await
2685            .expect("write b1 v2");
2686        s.sync().await.expect("sync #2");
2687
2688        let cm2 = get_commit_marker(db);
2689        assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2690        let meta2 = s.get_block_metadata_for_testing();
2691        assert_eq!(
2692            meta2.get(&b1).unwrap().1 as u64,
2693            cm2,
2694            "updated block tracks new version"
2695        );
2696        assert_eq!(
2697            meta2.get(&b2).unwrap().1 as u64,
2698            1,
2699            "unchanged block retains prior version"
2700        );
2701    }
2702}