1#[allow(unused_imports)]
2use super::metadata::BlockMetadataPersist;
3use super::metadata::{ChecksumAlgorithm, ChecksumManager};
4#[cfg(any(
5 target_arch = "wasm32",
6 all(
7 not(target_arch = "wasm32"),
8 any(test, debug_assertions),
9 not(feature = "fs_persist")
10 )
11))]
12use super::vfs_sync;
13use crate::types::DatabaseError;
14#[cfg(target_arch = "wasm32")]
15use js_sys::Date;
16#[cfg(not(target_arch = "wasm32"))]
17use parking_lot::Mutex;
18#[cfg(target_arch = "wasm32")]
19use std::cell::RefCell;
20use std::collections::{HashMap, HashSet, VecDeque};
21#[allow(unused_imports)]
22use std::sync::{
23 Arc,
24 atomic::{AtomicBool, AtomicU64, Ordering},
25};
26use std::time::Duration;
27#[cfg(not(target_arch = "wasm32"))]
28use std::time::{Instant, SystemTime, UNIX_EPOCH};
29#[cfg(not(target_arch = "wasm32"))]
30use tokio::sync::mpsc;
31#[cfg(not(target_arch = "wasm32"))]
32use tokio::task::JoinHandle as TokioJoinHandle;
33
34#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
36use std::path::PathBuf;
37
38#[cfg(not(target_arch = "wasm32"))]
44#[derive(Debug)]
45pub(super) enum SyncRequest {
46 Timer(tokio::sync::oneshot::Sender<()>),
47 Debounce(tokio::sync::oneshot::Sender<()>),
48}
49
50#[derive(Clone, Debug, Default)]
51pub struct RecoveryOptions {
52 pub mode: RecoveryMode,
53 pub on_corruption: CorruptionAction,
54}
55
56#[derive(Clone, Debug, Default)]
57pub enum RecoveryMode {
58 #[default]
59 Full,
60 Sample {
61 count: usize,
62 },
63 Skip,
64}
65
66#[derive(Clone, Debug, Default)]
67pub enum CorruptionAction {
68 #[default]
69 Report,
70 Repair,
71 Fail,
72}
73
74#[derive(Clone, Debug, PartialEq)]
75pub enum CrashRecoveryAction {
76 NoActionNeeded,
77 Rollback,
78 Finalize,
79}
80
81#[derive(Clone, Debug, Default)]
82pub struct RecoveryReport {
83 pub total_blocks_verified: usize,
84 pub corrupted_blocks: Vec<u64>,
85 pub repaired_blocks: Vec<u64>,
86 pub verification_duration_ms: u64,
87}
88
89#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
91#[derive(serde::Serialize, serde::Deserialize, Default)]
92#[allow(dead_code)]
93struct FsMeta {
94 entries: Vec<(u64, BlockMetadataPersist)>,
95}
96
97#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
98#[derive(serde::Serialize, serde::Deserialize, Default)]
99#[allow(dead_code)]
100struct FsAlloc {
101 allocated: Vec<u64>,
102}
103
104#[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
105#[derive(serde::Serialize, serde::Deserialize, Default)]
106#[allow(dead_code)]
107struct FsDealloc {
108 tombstones: Vec<u64>,
109}
110
111#[cfg(not(target_arch = "wasm32"))]
113thread_local! {
114 pub(super) static GLOBAL_METADATA_TEST: parking_lot::Mutex<HashMap<String, HashMap<u64, BlockMetadataPersist>>> = parking_lot::Mutex::new(HashMap::new());
115}
116
117#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
119thread_local! {
120 static GLOBAL_COMMIT_MARKER_TEST: parking_lot::Mutex<HashMap<String, u64>> = parking_lot::Mutex::new(HashMap::new());
121}
122
123#[derive(Clone, Debug)]
124pub struct SyncPolicy {
125 pub interval_ms: Option<u64>,
126 pub max_dirty: Option<usize>,
127 pub max_dirty_bytes: Option<usize>,
128 pub debounce_ms: Option<u64>,
129 pub verify_after_write: bool,
130}
131
132#[cfg(not(target_arch = "wasm32"))]
133impl Drop for BlockStorage {
134 fn drop(&mut self) {
135 if let Some(stop) = &self.auto_sync_stop {
136 stop.store(true, Ordering::SeqCst);
137 }
138 if let Some(handle) = self.auto_sync_thread.take() {
139 let _ = handle.join();
140 }
141 if let Some(handle) = self.debounce_thread.take() {
142 let _ = handle.join();
143 }
144 if let Some(task) = self.tokio_timer_task.take() {
145 task.abort();
146 }
147 if let Some(task) = self.tokio_debounce_task.take() {
148 task.abort();
149 }
150 self.auto_sync_stop = None;
151 }
152}
153
154pub const BLOCK_SIZE: usize = 4096;
155#[allow(dead_code)]
156pub(super) const DEFAULT_CACHE_CAPACITY: usize = 128;
157#[allow(dead_code)]
158const STORE_NAME: &str = "sqlite_blocks";
159#[allow(dead_code)]
160const METADATA_STORE: &str = "metadata";
161
162#[cfg(target_arch = "wasm32")]
165macro_rules! lock_mutex {
166 ($mutex:expr) => {
167 $mutex
168 .try_borrow_mut()
169 .expect("RefCell borrow failed - reentrancy detected in block_storage.rs")
170 };
171}
172
173#[allow(unused_macros)]
175#[cfg(target_arch = "wasm32")]
176macro_rules! try_lock_mutex {
177 ($mutex:expr) => {
178 $mutex
179 };
180}
181
182#[cfg(not(target_arch = "wasm32"))]
183macro_rules! lock_mutex {
184 ($mutex:expr) => {
185 $mutex.lock()
186 };
187}
188
189#[allow(unused_macros)]
191#[cfg(target_arch = "wasm32")]
192macro_rules! read_lock {
193 ($mutex:expr) => {
194 $mutex
195 .try_borrow()
196 .expect("RefCell borrow failed - likely reentrancy issue")
197 };
198}
199
200#[allow(unused_macros)]
201#[cfg(not(target_arch = "wasm32"))]
202macro_rules! read_lock {
203 ($mutex:expr) => {
204 $mutex.lock()
205 };
206}
207
208#[allow(unused_macros)]
210#[cfg(target_arch = "wasm32")]
211macro_rules! try_lock_mutex {
212 ($mutex:expr) => {
213 $mutex.ok()
214 };
215}
216
217#[allow(unused_macros)]
218#[cfg(not(target_arch = "wasm32"))]
219macro_rules! try_lock_mutex {
220 ($mutex:expr) => {
221 Some($mutex.lock())
222 };
223}
224
225#[allow(unused_macros)]
227#[cfg(target_arch = "wasm32")]
228macro_rules! try_read_lock {
229 ($mutex:expr) => {
230 $mutex.try_borrow().ok()
231 };
232}
233
234#[allow(unused_macros)]
235#[cfg(not(target_arch = "wasm32"))]
236macro_rules! try_read_lock {
237 ($mutex:expr) => {
238 Some($mutex.lock())
239 };
240}
241
242#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
243pub enum StorageBackend {
244 #[default]
245 IndexedDb,
246 Opfs,
247 Hybrid,
248}
249
250impl StorageBackend {
251 pub fn as_str(self) -> &'static str {
252 match self {
253 StorageBackend::IndexedDb => "IndexedDB",
254 StorageBackend::Opfs => "OPFS",
255 StorageBackend::Hybrid => "Hybrid",
256 }
257 }
258}
259
260pub struct BlockStorage {
261 #[cfg(target_arch = "wasm32")]
264 pub(super) cache: RefCell<HashMap<u64, Vec<u8>>>,
265 #[cfg(not(target_arch = "wasm32"))]
266 pub(super) cache: Mutex<HashMap<u64, Vec<u8>>>,
267
268 #[cfg(target_arch = "wasm32")]
269 pub(super) dirty_blocks: std::rc::Rc<RefCell<HashMap<u64, Vec<u8>>>>,
270 #[cfg(not(target_arch = "wasm32"))]
271 pub(super) dirty_blocks: Arc<Mutex<HashMap<u64, Vec<u8>>>>,
272
273 #[cfg(target_arch = "wasm32")]
274 pub(super) allocated_blocks: RefCell<HashSet<u64>>,
275 #[cfg(not(target_arch = "wasm32"))]
276 pub(super) allocated_blocks: Mutex<HashSet<u64>>,
277
278 #[cfg(target_arch = "wasm32")]
279 #[allow(dead_code)]
280 pub(super) deallocated_blocks: RefCell<HashSet<u64>>,
281 #[cfg(not(target_arch = "wasm32"))]
282 #[allow(dead_code)]
283 pub(super) deallocated_blocks: Mutex<HashSet<u64>>,
284 pub(super) next_block_id: AtomicU64,
285 pub(super) capacity: usize,
286
287 #[cfg(target_arch = "wasm32")]
288 pub(super) lru_order: RefCell<VecDeque<u64>>,
289 #[cfg(not(target_arch = "wasm32"))]
290 pub(super) lru_order: Mutex<VecDeque<u64>>,
291
292 pub(super) checksum_manager: ChecksumManager,
294 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
295 pub(super) base_dir: PathBuf,
296 pub(super) db_name: String,
297 pub(super) backend: StorageBackend,
298
299 #[cfg(target_arch = "wasm32")]
301 pub(super) auto_sync_interval: RefCell<Option<Duration>>,
302 #[cfg(not(target_arch = "wasm32"))]
303 pub(super) auto_sync_interval: Mutex<Option<Duration>>,
304
305 #[cfg(not(target_arch = "wasm32"))]
306 pub(super) last_auto_sync: Instant,
307
308 #[cfg(target_arch = "wasm32")]
309 pub(super) policy: RefCell<Option<SyncPolicy>>,
310 #[cfg(not(target_arch = "wasm32"))]
311 pub(super) policy: Mutex<Option<SyncPolicy>>,
312 #[cfg(not(target_arch = "wasm32"))]
313 pub(super) auto_sync_stop: Option<Arc<AtomicBool>>,
314 #[cfg(not(target_arch = "wasm32"))]
315 pub(super) auto_sync_thread: Option<std::thread::JoinHandle<()>>,
316 #[cfg(not(target_arch = "wasm32"))]
317 pub(super) debounce_thread: Option<std::thread::JoinHandle<()>>,
318 #[cfg(not(target_arch = "wasm32"))]
319 pub(super) tokio_timer_task: Option<TokioJoinHandle<()>>,
320 #[cfg(not(target_arch = "wasm32"))]
321 pub(super) tokio_debounce_task: Option<TokioJoinHandle<()>>,
322 #[cfg(not(target_arch = "wasm32"))]
323 pub(super) last_write_ms: Arc<AtomicU64>,
324 #[cfg(not(target_arch = "wasm32"))]
325 pub(super) threshold_hit: Arc<AtomicBool>,
326 #[cfg(not(target_arch = "wasm32"))]
327 pub(super) sync_count: Arc<AtomicU64>,
328 #[cfg(not(target_arch = "wasm32"))]
329 pub(super) timer_sync_count: Arc<AtomicU64>,
330 #[cfg(not(target_arch = "wasm32"))]
331 pub(super) debounce_sync_count: Arc<AtomicU64>,
332 #[cfg(not(target_arch = "wasm32"))]
333 pub(super) last_sync_duration_ms: Arc<AtomicU64>,
334
335 #[cfg(not(target_arch = "wasm32"))]
337 pub(super) sync_sender: Option<mpsc::UnboundedSender<SyncRequest>>,
338 #[cfg(not(target_arch = "wasm32"))]
339 pub(super) sync_receiver: Option<mpsc::UnboundedReceiver<SyncRequest>>,
340
341 pub(super) recovery_report: RecoveryReport,
343
344 #[cfg(target_arch = "wasm32")]
346 pub leader_election: std::cell::RefCell<Option<super::leader_election::LeaderElectionManager>>,
347
348 pub(super) observability: super::observability::ObservabilityManager,
350
351 #[cfg(all(feature = "telemetry", not(target_arch = "wasm32")))]
353 pub(super) metrics: Option<crate::telemetry::Metrics>,
354 #[cfg(all(feature = "telemetry", target_arch = "wasm32"))]
355 pub(super) metrics: RefCell<Option<crate::telemetry::Metrics>>,
356}
357
358impl BlockStorage {
359 #[cfg(target_arch = "wasm32")]
362 pub fn new_sync(db_name: &str) -> Self {
363 log::info!(
364 "Creating BlockStorage synchronously for database: {}",
365 db_name
366 );
367
368 use crate::storage::vfs_sync::with_global_storage;
370 let (cache, allocated_blocks, max_block_id) = with_global_storage(|storage_map| {
371 if let Some(db_storage) = storage_map.borrow().get(db_name) {
372 let cache = db_storage.clone();
373 let allocated = db_storage.keys().copied().collect::<HashSet<_>>();
374 let max_id = db_storage.keys().max().copied().unwrap_or(0);
375 (cache, allocated, max_id)
376 } else {
377 (HashMap::new(), HashSet::new(), 0)
378 }
379 });
380
381 log::info!(
382 "Loaded {} blocks from GLOBAL_STORAGE for {} (max_block_id={})",
383 cache.len(),
384 db_name,
385 max_block_id
386 );
387
388 use crate::storage::vfs_sync::with_global_metadata;
391 let checksum_manager = with_global_metadata(|metadata_map| {
392 if let Some(db_metadata) = metadata_map.borrow().get(db_name) {
393 let mut checksums = HashMap::new();
394 let mut algos = HashMap::new();
395 for (block_id, meta) in db_metadata {
396 checksums.insert(*block_id, meta.checksum);
397 algos.insert(*block_id, meta.algo);
398 }
399 ChecksumManager::with_data(checksums, algos, ChecksumAlgorithm::FastHash)
400 } else {
401 ChecksumManager::new(ChecksumAlgorithm::FastHash)
402 }
403 });
404
405 Self {
406 cache: RefCell::new(cache),
407 dirty_blocks: std::rc::Rc::new(RefCell::new(HashMap::new())),
408 allocated_blocks: RefCell::new(allocated_blocks),
409 deallocated_blocks: RefCell::new(HashSet::new()),
410 next_block_id: AtomicU64::new(max_block_id + 1),
411 capacity: 128,
412 lru_order: RefCell::new(VecDeque::new()),
413 checksum_manager,
414 db_name: db_name.to_string(),
415 backend: StorageBackend::IndexedDb,
416 auto_sync_interval: RefCell::new(None),
417 policy: RefCell::new(None),
418 #[cfg(not(target_arch = "wasm32"))]
419 last_auto_sync: Instant::now(),
420 #[cfg(not(target_arch = "wasm32"))]
421 auto_sync_stop: None,
422 #[cfg(not(target_arch = "wasm32"))]
423 auto_sync_thread: None,
424 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
425 base_dir: std::path::PathBuf::from(
426 std::env::var("ABSURDERSQL_FS_BASE")
427 .unwrap_or_else(|_| "./test_storage".to_string()),
428 ),
429 #[cfg(not(target_arch = "wasm32"))]
430 debounce_thread: None,
431 #[cfg(not(target_arch = "wasm32"))]
432 tokio_timer_task: None,
433 #[cfg(not(target_arch = "wasm32"))]
434 tokio_debounce_task: None,
435 #[cfg(not(target_arch = "wasm32"))]
436 last_write_ms: Arc::new(AtomicU64::new(0)),
437 #[cfg(not(target_arch = "wasm32"))]
438 threshold_hit: Arc::new(AtomicBool::new(false)),
439 #[cfg(not(target_arch = "wasm32"))]
440 sync_count: Arc::new(AtomicU64::new(0)),
441 #[cfg(not(target_arch = "wasm32"))]
442 timer_sync_count: Arc::new(AtomicU64::new(0)),
443 #[cfg(not(target_arch = "wasm32"))]
444 debounce_sync_count: Arc::new(AtomicU64::new(0)),
445 #[cfg(not(target_arch = "wasm32"))]
446 last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
447 #[cfg(not(target_arch = "wasm32"))]
448 sync_sender: None,
449 #[cfg(not(target_arch = "wasm32"))]
450 sync_receiver: None,
451 recovery_report: RecoveryReport::default(),
452 #[cfg(target_arch = "wasm32")]
453 leader_election: std::cell::RefCell::new(None),
454 observability: super::observability::ObservabilityManager::new(),
455 #[cfg(feature = "telemetry")]
456 metrics: Default::default(),
457 }
458 }
459
460 #[cfg(target_arch = "wasm32")]
461 pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
462 super::constructors::new_wasm(db_name).await
463 }
464
465 #[cfg(target_arch = "wasm32")]
466 pub async fn new_with_backend(
467 db_name: &str,
468 backend: StorageBackend,
469 ) -> Result<Self, DatabaseError> {
470 super::constructors::new_wasm_with_backend(db_name, backend).await
471 }
472
473 #[cfg(not(target_arch = "wasm32"))]
474 pub async fn new(db_name: &str) -> Result<Self, DatabaseError> {
475 let db_name = crate::utils::normalize_db_name(db_name);
477 let db_name = db_name.as_str();
478 log::info!("Creating BlockStorage for database: {}", db_name);
479
480 let (allocated_blocks, next_block_id) = {
482 #[cfg(feature = "fs_persist")]
483 {
484 let mut allocated_blocks = HashSet::new();
486 let mut next_block_id: u64 = 1;
487
488 let base_path = std::env::var("ABSURDERSQL_FS_BASE")
489 .unwrap_or_else(|_| "./test_storage".to_string());
490 let mut alloc_path = std::path::PathBuf::from(base_path);
491 alloc_path.push(db_name);
492 alloc_path.push("allocations.json");
493
494 if let Ok(content) = std::fs::read_to_string(&alloc_path) {
495 if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
496 if let Some(allocated_array) = alloc_data["allocated"].as_array() {
497 for block_id_val in allocated_array {
498 if let Some(block_id) = block_id_val.as_u64() {
499 allocated_blocks.insert(block_id);
500 }
501 }
502 next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
503 }
504 }
505 }
506
507 (allocated_blocks, next_block_id)
508 }
509
510 #[cfg(not(feature = "fs_persist"))]
511 {
512 let mut allocated_blocks = HashSet::new();
514 let mut next_block_id: u64 = 1;
515
516 super::vfs_sync::with_global_allocation_map(|allocation_map| {
517 if let Some(existing_allocations) = allocation_map.borrow().get(db_name) {
518 allocated_blocks = existing_allocations.clone();
519 next_block_id = allocated_blocks.iter().max().copied().unwrap_or(0) + 1;
520 log::info!(
521 "Restored {} allocated blocks for database: {}",
522 allocated_blocks.len(),
523 db_name
524 );
525 }
526 });
527
528 (allocated_blocks, next_block_id)
529 }
530 };
531
532 let checksums_init: HashMap<u64, u64> = {
534 #[cfg(feature = "fs_persist")]
535 {
536 let mut map = HashMap::new();
537 let base_path = std::env::var("ABSURDERSQL_FS_BASE")
538 .unwrap_or_else(|_| "./test_storage".to_string());
539 let mut meta_path = std::path::PathBuf::from(base_path);
540 meta_path.push(db_name);
541 meta_path.push("metadata.json");
542
543 if let Ok(content) = std::fs::read_to_string(&meta_path) {
544 if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
545 if let Some(entries) = meta_data["entries"].as_array() {
546 for entry in entries {
547 if let Some(arr) = entry.as_array() {
548 if let (Some(block_id), Some(obj)) = (
549 arr.first().and_then(|v| v.as_u64()),
550 arr.get(1).and_then(|v| v.as_object()),
551 ) {
552 if let Some(checksum) =
553 obj.get("checksum").and_then(|v| v.as_u64())
554 {
555 map.insert(block_id, checksum);
556 }
557 }
558 }
559 }
560 }
561 }
562 }
563 map
564 }
565
566 #[cfg(not(feature = "fs_persist"))]
567 {
568 let mut map = HashMap::new();
570 GLOBAL_METADATA_TEST.with(|meta| {
571 let meta_map = meta.lock();
572 if let Some(db_meta) = meta_map.get(db_name) {
573 for (block_id, metadata) in db_meta.iter() {
574 map.insert(*block_id, metadata.checksum);
575 }
576 }
577 });
578 map
579 }
580 };
581
582 let checksum_algos_init: HashMap<u64, ChecksumAlgorithm> = {
583 #[cfg(feature = "fs_persist")]
584 {
585 let mut map = HashMap::new();
586 let base_path = std::env::var("ABSURDERSQL_FS_BASE")
587 .unwrap_or_else(|_| "./test_storage".to_string());
588 let mut meta_path = std::path::PathBuf::from(base_path);
589 meta_path.push(db_name);
590 meta_path.push("metadata.json");
591
592 if let Ok(content) = std::fs::read_to_string(&meta_path) {
593 if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
594 if let Some(entries) = meta_data["entries"].as_array() {
595 for entry in entries {
596 if let Some(arr) = entry.as_array() {
597 if let (Some(block_id), Some(obj)) = (
598 arr.first().and_then(|v| v.as_u64()),
599 arr.get(1).and_then(|v| v.as_object()),
600 ) {
601 let algo = obj
602 .get("algo")
603 .and_then(|v| v.as_str())
604 .and_then(|s| match s {
605 "CRC32" => Some(ChecksumAlgorithm::CRC32),
606 "FastHash" => Some(ChecksumAlgorithm::FastHash),
607 _ => None,
608 })
609 .unwrap_or(ChecksumAlgorithm::FastHash);
610 map.insert(block_id, algo);
611 }
612 }
613 }
614 }
615 }
616 }
617 map
618 }
619
620 #[cfg(not(feature = "fs_persist"))]
621 {
622 let mut map = HashMap::new();
624 GLOBAL_METADATA_TEST.with(|meta| {
625 let meta_map = meta.lock();
626 if let Some(db_meta) = meta_map.get(db_name) {
627 for (block_id, metadata) in db_meta.iter() {
628 map.insert(*block_id, metadata.algo);
629 }
630 }
631 });
632 map
633 }
634 };
635
636 #[cfg(feature = "fs_persist")]
638 let checksum_algo_default = match std::env::var("DATASYNC_CHECKSUM_ALGO").ok().as_deref() {
639 Some("CRC32") => ChecksumAlgorithm::CRC32,
640 _ => ChecksumAlgorithm::FastHash,
641 };
642 #[cfg(not(feature = "fs_persist"))]
643 let checksum_algo_default = ChecksumAlgorithm::FastHash;
644
645 let deallocated_blocks_init: HashSet<u64> = {
647 #[cfg(feature = "fs_persist")]
648 {
649 let mut set = HashSet::new();
650 let base_path = std::env::var("ABSURDERSQL_FS_BASE")
651 .unwrap_or_else(|_| "./test_storage".to_string());
652 let mut path = std::path::PathBuf::from(base_path);
653 path.push(db_name);
654 let mut dealloc_path = path.clone();
655 dealloc_path.push("deallocated.json");
656 if let Ok(content) = std::fs::read_to_string(&dealloc_path) {
657 if let Ok(dealloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
658 if let Some(tombstones_array) = dealloc_data["tombstones"].as_array() {
659 for tombstone_val in tombstones_array {
660 if let Some(block_id) = tombstone_val.as_u64() {
661 set.insert(block_id);
662 }
663 }
664 }
665 }
666 }
667 set
668 }
669 #[cfg(not(feature = "fs_persist"))]
670 {
671 HashSet::new()
672 }
673 };
674
675 Ok(BlockStorage {
676 db_name: db_name.to_string(),
677 backend: StorageBackend::IndexedDb,
678 cache: Mutex::new(HashMap::new()),
679 lru_order: Mutex::new(VecDeque::new()),
680 capacity: 1000,
681 checksum_manager: ChecksumManager::with_data(
682 checksums_init,
683 checksum_algos_init,
684 checksum_algo_default,
685 ),
686 dirty_blocks: Arc::new(Mutex::new(HashMap::new())),
687 allocated_blocks: Mutex::new(allocated_blocks),
688 next_block_id: AtomicU64::new(next_block_id),
689 deallocated_blocks: Mutex::new(deallocated_blocks_init),
690 policy: Mutex::new(None),
691 auto_sync_interval: Mutex::new(None),
692 #[cfg(not(target_arch = "wasm32"))]
693 last_auto_sync: Instant::now(),
694 #[cfg(not(target_arch = "wasm32"))]
695 auto_sync_stop: None,
696 #[cfg(not(target_arch = "wasm32"))]
697 auto_sync_thread: None,
698 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
699 base_dir: std::path::PathBuf::from(
700 std::env::var("ABSURDERSQL_FS_BASE")
701 .unwrap_or_else(|_| "./test_storage".to_string()),
702 ),
703 #[cfg(not(target_arch = "wasm32"))]
704 debounce_thread: None,
705 #[cfg(not(target_arch = "wasm32"))]
706 tokio_timer_task: None,
707 #[cfg(not(target_arch = "wasm32"))]
708 tokio_debounce_task: None,
709 #[cfg(not(target_arch = "wasm32"))]
710 last_write_ms: Arc::new(AtomicU64::new(0)),
711 #[cfg(not(target_arch = "wasm32"))]
712 threshold_hit: Arc::new(AtomicBool::new(false)),
713 #[cfg(not(target_arch = "wasm32"))]
714 sync_count: Arc::new(AtomicU64::new(0)),
715 #[cfg(not(target_arch = "wasm32"))]
716 timer_sync_count: Arc::new(AtomicU64::new(0)),
717 #[cfg(not(target_arch = "wasm32"))]
718 debounce_sync_count: Arc::new(AtomicU64::new(0)),
719 #[cfg(not(target_arch = "wasm32"))]
720 last_sync_duration_ms: Arc::new(AtomicU64::new(0)),
721 #[cfg(not(target_arch = "wasm32"))]
722 sync_sender: None,
723 #[cfg(not(target_arch = "wasm32"))]
724 sync_receiver: None,
725 recovery_report: RecoveryReport::default(),
726 #[cfg(target_arch = "wasm32")]
727 leader_election: std::cell::RefCell::new(None),
728 observability: super::observability::ObservabilityManager::new(),
729 #[cfg(feature = "telemetry")]
730 metrics: Default::default(),
731 })
732 }
733
734 pub async fn new_with_capacity(db_name: &str, capacity: usize) -> Result<Self, DatabaseError> {
735 let mut s = Self::new(db_name).await?;
736 s.capacity = capacity;
737 Ok(s)
738 }
739
740 pub fn storage_backend(&self) -> StorageBackend {
741 self.backend
742 }
743
744 pub async fn new_with_recovery_options(
745 db_name: &str,
746 recovery_opts: RecoveryOptions,
747 ) -> Result<Self, DatabaseError> {
748 let mut storage = Self::new(db_name).await?;
749
750 storage.perform_startup_recovery(recovery_opts).await?;
751
752 Ok(storage)
753 }
754
755 pub fn get_recovery_report(&self) -> &RecoveryReport {
756 &self.recovery_report
757 }
758
759 async fn perform_startup_recovery(
760 &mut self,
761 opts: RecoveryOptions,
762 ) -> Result<(), DatabaseError> {
763 super::recovery::perform_startup_recovery(self, opts).await
764 }
765
766 pub(super) async fn get_blocks_for_verification(
767 &self,
768 mode: &RecoveryMode,
769 ) -> Result<Vec<u64>, DatabaseError> {
770 let all_blocks: Vec<u64> = lock_mutex!(self.allocated_blocks).iter().copied().collect();
771
772 match mode {
773 RecoveryMode::Full => Ok(all_blocks),
774 RecoveryMode::Sample { count } => {
775 let sample_count = (*count).min(all_blocks.len());
776 let mut sampled = all_blocks;
777 sampled.sort_unstable(); sampled.truncate(sample_count);
779 Ok(sampled)
780 }
781 RecoveryMode::Skip => Ok(Vec::new()),
782 }
783 }
784
785 pub(super) async fn verify_block_integrity(
786 &mut self,
787 block_id: u64,
788 ) -> Result<bool, DatabaseError> {
789 let data = match self.read_block_from_storage(block_id).await {
791 Ok(data) => data,
792 Err(_) => {
793 log::warn!(
794 "Could not read block {} for integrity verification",
795 block_id
796 );
797 return Ok(false);
798 }
799 };
800
801 match self.verify_against_stored_checksum(block_id, &data) {
803 Ok(()) => Ok(true),
804 Err(e) => {
805 log::warn!(
806 "Block {} failed checksum verification: {}",
807 block_id,
808 e.message
809 );
810 Ok(false)
811 }
812 }
813 }
814
815 async fn read_block_from_storage(&mut self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
816 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
818 {
819 let mut blocks_dir = self.base_dir.clone();
820 blocks_dir.push(&self.db_name);
821 blocks_dir.push("blocks");
822 let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
823
824 if let Ok(data) = std::fs::read(&block_file) {
825 if data.len() == BLOCK_SIZE {
826 return Ok(data);
827 }
828 }
829 }
830
831 #[cfg(all(
833 not(target_arch = "wasm32"),
834 any(test, debug_assertions),
835 not(feature = "fs_persist")
836 ))]
837 {
838 let mut found_data = None;
839 vfs_sync::with_global_storage(|storage| {
840 #[cfg(target_arch = "wasm32")]
841 let storage_map = storage;
842 #[cfg(not(target_arch = "wasm32"))]
843 let storage_map = storage.borrow();
844 if let Some(db_storage) = storage_map.get(&self.db_name) {
845 if let Some(data) = db_storage.get(&block_id) {
846 found_data = Some(data.clone());
847 }
848 }
849 });
850 if let Some(data) = found_data {
851 return Ok(data);
852 }
853 }
854
855 #[cfg(target_arch = "wasm32")]
857 {
858 let mut found_data = None;
859 vfs_sync::with_global_storage(|storage_map| {
860 if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
861 if let Some(data) = db_storage.get(&block_id) {
862 found_data = Some(data.clone());
863 }
864 }
865 });
866 if let Some(data) = found_data {
867 return Ok(data);
868 }
869 }
870
871 Err(DatabaseError::new(
872 "BLOCK_NOT_FOUND",
873 &format!("Block {} not found in storage", block_id),
874 ))
875 }
876
877 pub(super) async fn repair_corrupted_block(
878 &mut self,
879 block_id: u64,
880 ) -> Result<bool, DatabaseError> {
881 log::info!("Attempting to repair corrupted block {}", block_id);
882
883 lock_mutex!(self.cache).remove(&block_id);
888
889 self.checksum_manager.remove_checksum(block_id);
891
892 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
894 {
895 let mut blocks_dir = self.base_dir.clone();
896 blocks_dir.push(&self.db_name);
897 blocks_dir.push("blocks");
898 let block_file = blocks_dir.join(format!("block_{}.bin", block_id));
899 let _ = std::fs::remove_file(&block_file);
900 }
901
902 #[cfg(all(
904 not(target_arch = "wasm32"),
905 any(test, debug_assertions),
906 not(feature = "fs_persist")
907 ))]
908 {
909 vfs_sync::with_global_storage(|storage| {
910 let mut storage_map = storage.borrow_mut();
911 if let Some(db_storage) = storage_map.get_mut(&self.db_name) {
912 db_storage.remove(&block_id);
913 }
914 });
915 }
916
917 #[cfg(target_arch = "wasm32")]
919 {
920 vfs_sync::with_global_storage(|storage_map| {
921 if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
922 db_storage.remove(&block_id);
923 }
924 });
925 }
926
927 log::info!(
928 "Corrupted block {} has been removed (repair completed)",
929 block_id
930 );
931 Ok(true)
932 }
933
934 pub(super) fn touch_lru(&self, block_id: u64) {
935 let mut lru = lock_mutex!(self.lru_order);
937 if let Some(pos) = lru.iter().position(|&id| id == block_id) {
938 lru.remove(pos);
939 }
940 lru.push_back(block_id);
942 }
943
944 pub(super) fn evict_if_needed(&self) {
945 loop {
947 let victim_opt = {
949 let cache_guard = lock_mutex!(self.cache);
950 if cache_guard.len() <= self.capacity {
951 break; }
953
954 let dirty_guard = lock_mutex!(self.dirty_blocks);
956 let mut lru_guard = lock_mutex!(self.lru_order);
957
958 let victim_pos = lru_guard
959 .iter()
960 .position(|id| !dirty_guard.contains_key(id));
961
962 victim_pos.map(|pos| lru_guard.remove(pos).expect("valid pos"))
963 };
965
966 match victim_opt {
967 Some(victim) => {
968 lock_mutex!(self.cache).remove(&victim);
970 }
971 None => {
972 break;
974 }
975 }
976 }
977 }
978
979 #[inline]
980 #[cfg(target_arch = "wasm32")]
981 pub fn now_millis() -> u64 {
982 Date::now() as u64
984 }
985
986 #[inline]
987 #[cfg(not(target_arch = "wasm32"))]
988 pub fn now_millis() -> u64 {
989 let now = SystemTime::now()
990 .duration_since(UNIX_EPOCH)
991 .unwrap_or_else(|_| Duration::from_millis(0));
992 now.as_millis() as u64
993 }
994
995 pub(super) fn verify_against_stored_checksum(
996 &self,
997 block_id: u64,
998 data: &[u8],
999 ) -> Result<(), DatabaseError> {
1000 self.checksum_manager.validate_checksum(block_id, data)
1001 }
1002
1003 pub fn read_block_sync(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
1005 super::io_operations::read_block_sync_impl(self, block_id)
1007 }
1008
1009 pub async fn read_block(&self, block_id: u64) -> Result<Vec<u8>, DatabaseError> {
1010 self.read_block_sync(block_id)
1012 }
1013
1014 #[cfg(target_arch = "wasm32")]
1016 pub fn write_block_sync(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1017 super::io_operations::write_block_sync_impl(self, block_id, data)
1018 }
1019
1020 #[cfg(not(target_arch = "wasm32"))]
1021 pub fn write_block_sync(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1022 super::io_operations::write_block_sync_impl(self, block_id, data)
1023 }
1024
1025 #[cfg(target_arch = "wasm32")]
1026 pub async fn write_block(&self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1027 self.write_block_sync(block_id, data)
1028 }
1029
1030 #[cfg(not(target_arch = "wasm32"))]
1031 pub async fn write_block(&mut self, block_id: u64, data: Vec<u8>) -> Result<(), DatabaseError> {
1032 self.write_block_sync(block_id, data)?;
1033
1034 if self.threshold_hit.load(std::sync::atomic::Ordering::SeqCst) {
1036 let has_debounce = lock_mutex!(self.policy)
1037 .as_ref()
1038 .and_then(|p| p.debounce_ms)
1039 .is_some();
1040 if !has_debounce {
1041 log::debug!("Threshold hit without debounce: performing inline sync");
1042 self.sync_implementation()?;
1043 self.threshold_hit
1044 .store(false, std::sync::atomic::Ordering::SeqCst);
1045 }
1046 }
1047
1048 Ok(())
1049 }
1050
1051 #[cfg(target_arch = "wasm32")]
1053 pub fn write_blocks_sync(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1054 self.maybe_auto_sync();
1055 for (block_id, data) in items {
1056 self.write_block_sync(block_id, data)?;
1057 }
1058 Ok(())
1059 }
1060
1061 #[cfg(not(target_arch = "wasm32"))]
1062 pub fn write_blocks_sync(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1063 self.maybe_auto_sync();
1064 for (block_id, data) in items {
1065 self.write_block_sync(block_id, data)?;
1066 }
1067 Ok(())
1068 }
1069
1070 #[cfg(target_arch = "wasm32")]
1072 pub async fn write_blocks(&self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1073 self.write_blocks_sync(items)
1074 }
1075
1076 #[cfg(not(target_arch = "wasm32"))]
1077 pub async fn write_blocks(&mut self, items: Vec<(u64, Vec<u8>)>) -> Result<(), DatabaseError> {
1078 self.write_blocks_sync(items)
1079 }
1080
1081 pub fn read_blocks_sync(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1083 self.maybe_auto_sync();
1084 let mut results = Vec::with_capacity(block_ids.len());
1085 for &id in block_ids {
1086 results.push(self.read_block_sync(id)?);
1087 }
1088 Ok(results)
1089 }
1090
1091 pub async fn read_blocks(&self, block_ids: &[u64]) -> Result<Vec<Vec<u8>>, DatabaseError> {
1093 self.read_blocks_sync(block_ids)
1094 }
1095
1096 pub fn get_block_checksum(&self, block_id: u64) -> Option<u32> {
1098 self.checksum_manager
1099 .get_checksum(block_id)
1100 .map(|checksum| checksum as u32)
1101 }
1102
1103 #[cfg(target_arch = "wasm32")]
1105 pub fn get_commit_marker(&self) -> u64 {
1106 vfs_sync::with_global_commit_marker(|cm| {
1107 cm.borrow().get(&self.db_name).copied().unwrap_or(0)
1108 })
1109 }
1110
1111 #[cfg(target_arch = "wasm32")]
1113 pub fn has_any_blocks(&self) -> bool {
1114 vfs_sync::with_global_storage(|storage_map| {
1115 storage_map
1116 .borrow()
1117 .get(&self.db_name)
1118 .is_some_and(|blocks| !blocks.is_empty())
1119 })
1120 }
1121
1122 pub async fn verify_block_checksum(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1123 if let Some(bytes) = lock_mutex!(self.cache).get(&block_id).cloned() {
1125 return self.verify_against_stored_checksum(block_id, &bytes);
1126 }
1127 let data = self.read_block_sync(block_id)?;
1129 self.verify_against_stored_checksum(block_id, &data)
1130 }
1131
1132 #[allow(unused_mut)]
1134 pub fn get_block_metadata_for_testing(&mut self) -> HashMap<u64, (u64, u32, u64)> {
1135 #[cfg(target_arch = "wasm32")]
1137 {
1138 let mut out = HashMap::new();
1139 vfs_sync::with_global_metadata(|meta_map| {
1140 if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1141 for (bid, m) in db_meta.iter() {
1142 out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1143 }
1144 }
1145 });
1146 out
1147 }
1148 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
1149 {
1150 use super::fs_persist::FsMeta;
1151 use std::io::Read;
1152 let mut out = HashMap::new();
1153 let base: PathBuf = self.base_dir.clone();
1154 let mut db_dir = base.clone();
1155 db_dir.push(&self.db_name);
1156 let mut meta_path = db_dir.clone();
1157 meta_path.push("metadata.json");
1158 if let Ok(mut f) = std::fs::File::open(&meta_path) {
1159 let mut s = String::new();
1160 if f.read_to_string(&mut s).is_ok() {
1161 if let Ok(meta) = serde_json::from_str::<FsMeta>(&s) {
1162 for (block_id, metadata) in meta.entries {
1163 out.insert(
1164 block_id,
1165 (
1166 metadata.checksum,
1167 metadata.version,
1168 metadata.last_modified_ms,
1169 ),
1170 );
1171 }
1172 }
1173 }
1174 }
1175 out
1176 }
1177 #[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
1178 {
1179 let mut out = HashMap::new();
1180 GLOBAL_METADATA_TEST.with(|meta| {
1181 let meta_map = meta.lock();
1182 if let Some(db_meta) = meta_map.get(&self.db_name) {
1183 for (bid, m) in db_meta.iter() {
1184 out.insert(*bid, (m.checksum, m.version, m.last_modified_ms));
1185 }
1186 }
1187 });
1188 out
1189 }
1190 }
1191
1192 pub fn set_block_checksum_for_testing(&mut self, block_id: u64, checksum: u64) {
1194 self.checksum_manager
1195 .set_checksum_for_testing(block_id, checksum);
1196 }
1197
1198 #[cfg(not(target_arch = "wasm32"))]
1200 pub(super) fn get_dirty_blocks(&self) -> &Arc<Mutex<HashMap<u64, Vec<u8>>>> {
1201 &self.dirty_blocks
1202 }
1203
1204 #[cfg(target_arch = "wasm32")]
1205 pub async fn sync(&self) -> Result<(), DatabaseError> {
1206 self.sync_async().await
1207 }
1208
1209 #[cfg(not(target_arch = "wasm32"))]
1210 pub async fn sync(&mut self) -> Result<(), DatabaseError> {
1211 self.sync_implementation()
1212 }
1213
1214 #[cfg(target_arch = "wasm32")]
1216 #[allow(invalid_reference_casting)]
1217 pub fn sync_now(&self) -> Result<(), DatabaseError> {
1218 let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1221 self_mut.sync_implementation()
1222 }
1223
1224 #[cfg(not(target_arch = "wasm32"))]
1225 pub fn sync_now(&mut self) -> Result<(), DatabaseError> {
1226 self.sync_implementation()
1227 }
1228
1229 fn sync_implementation(&mut self) -> Result<(), DatabaseError> {
1231 super::sync_operations::sync_implementation_impl(self)
1232 }
1233
1234 #[cfg(target_arch = "wasm32")]
1237 pub fn sync_blocks_only(&mut self) -> Result<(), DatabaseError> {
1238 super::wasm_vfs_sync::sync_blocks_only(self)
1239 }
1240
1241 #[cfg(target_arch = "wasm32")]
1243 pub async fn sync_async(&self) -> Result<(), DatabaseError> {
1244 super::wasm_indexeddb::sync_async(self).await
1245 }
1246
1247 #[cfg(target_arch = "wasm32")]
1250 #[allow(invalid_reference_casting)]
1251 pub fn drain_and_shutdown(&self) {
1252 let self_mut = unsafe { &mut *(self as *const Self as *mut Self) };
1254 if let Err(e) = self_mut.sync_now() {
1255 log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1256 }
1257 *lock_mutex!(self.auto_sync_interval) = None;
1258 }
1259
1260 #[cfg(not(target_arch = "wasm32"))]
1261 pub fn drain_and_shutdown(&mut self) {
1262 if let Err(e) = self.sync_now() {
1263 log::error!("drain_and_shutdown: sync_now failed: {}", e.message);
1264 }
1265 *lock_mutex!(self.auto_sync_interval) = None;
1266 #[cfg(not(target_arch = "wasm32"))]
1267 {
1268 if let Some(stop) = &self.auto_sync_stop {
1269 stop.store(true, Ordering::SeqCst);
1270 }
1271 if let Some(handle) = self.auto_sync_thread.take() {
1272 let _ = handle.join();
1273 }
1274 if let Some(handle) = self.debounce_thread.take() {
1275 let _ = handle.join();
1276 }
1277 if let Some(task) = self.tokio_timer_task.take() {
1278 task.abort();
1279 }
1280 if let Some(task) = self.tokio_debounce_task.take() {
1281 task.abort();
1282 }
1283 self.auto_sync_stop = None;
1284 self.threshold_hit.store(false, Ordering::SeqCst);
1285 }
1286 }
1287
1288 pub fn clear_cache(&self) {
1289 lock_mutex!(self.cache).clear();
1290 lock_mutex!(self.lru_order).clear();
1291 }
1292
1293 pub async fn on_database_import(&mut self) -> Result<(), DatabaseError> {
1313 log::info!(
1314 "Clearing cache for database '{}' after import",
1315 self.db_name
1316 );
1317
1318 self.clear_cache();
1320
1321 lock_mutex!(self.dirty_blocks).clear();
1323
1324 self.checksum_manager.clear_checksums();
1326
1327 #[cfg(target_arch = "wasm32")]
1329 {
1330 use super::vfs_sync::with_global_allocation_map;
1331 *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1332 gam.borrow()
1333 .get(&self.db_name)
1334 .cloned()
1335 .unwrap_or_else(std::collections::HashSet::new)
1336 });
1337 log::debug!(
1338 "Reloaded {} allocated blocks from global allocation map",
1339 lock_mutex!(self.allocated_blocks).len()
1340 );
1341
1342 log::debug!("Checksum data will be reloaded from metadata on next verification");
1344 }
1345
1346 #[cfg(not(target_arch = "wasm32"))]
1347 {
1348 #[cfg(feature = "fs_persist")]
1349 {
1350 let mut alloc_path = self.base_dir.clone();
1352 alloc_path.push(&self.db_name);
1353 alloc_path.push("allocations.json");
1354
1355 if let Ok(content) = std::fs::read_to_string(&alloc_path) {
1356 if let Ok(alloc_data) = serde_json::from_str::<serde_json::Value>(&content) {
1357 if let Some(allocated_array) = alloc_data["allocated"].as_array() {
1358 lock_mutex!(self.allocated_blocks).clear();
1359 for block_id_val in allocated_array {
1360 if let Some(block_id) = block_id_val.as_u64() {
1361 lock_mutex!(self.allocated_blocks).insert(block_id);
1362 }
1363 }
1364 log::debug!(
1365 "Reloaded {} allocated blocks from filesystem",
1366 lock_mutex!(self.allocated_blocks).len()
1367 );
1368 }
1369 }
1370 }
1371
1372 let mut meta_path = self.base_dir.clone();
1374 meta_path.push(&self.db_name);
1375 meta_path.push("metadata.json");
1376
1377 if let Ok(content) = std::fs::read_to_string(&meta_path) {
1378 if let Ok(meta_data) = serde_json::from_str::<serde_json::Value>(&content) {
1379 if let Some(entries) = meta_data["entries"].as_array() {
1380 let mut new_checksums = HashMap::new();
1381 let mut new_algos = HashMap::new();
1382
1383 for entry in entries {
1384 if let (Some(block_id), Some(checksum), Some(algo_str)) = (
1385 entry[0].as_u64(),
1386 entry[1]["checksum"].as_u64(),
1387 entry[1]["algo"].as_str(),
1388 ) {
1389 new_checksums.insert(block_id, checksum);
1390
1391 let algo = match algo_str {
1392 "CRC32" => super::metadata::ChecksumAlgorithm::CRC32,
1393 _ => super::metadata::ChecksumAlgorithm::FastHash,
1394 };
1395 new_algos.insert(block_id, algo);
1396 }
1397 }
1398
1399 self.checksum_manager
1400 .replace_all(new_checksums.clone(), new_algos);
1401 log::debug!(
1402 "Reloaded {} checksums from filesystem metadata",
1403 new_checksums.len()
1404 );
1405 }
1406 }
1407 } else {
1408 log::debug!("No metadata file found, checksums will be empty after import");
1409 }
1410 }
1411
1412 #[cfg(not(feature = "fs_persist"))]
1413 {
1414 use super::vfs_sync::with_global_allocation_map;
1416
1417 *lock_mutex!(self.allocated_blocks) = with_global_allocation_map(|gam| {
1418 #[cfg(target_arch = "wasm32")]
1419 let map = gam;
1420 #[cfg(not(target_arch = "wasm32"))]
1421 let map = gam.borrow();
1422 map.get(&self.db_name)
1423 .cloned()
1424 .unwrap_or_else(std::collections::HashSet::new)
1425 });
1426 log::debug!(
1427 "Reloaded {} allocated blocks from global allocation map (native test)",
1428 lock_mutex!(self.allocated_blocks).len()
1429 );
1430
1431 log::debug!("Checksum data will be reloaded from metadata on next verification");
1433 }
1434 }
1435
1436 log::info!(
1437 "Cache and allocation state refreshed for '{}'",
1438 self.db_name
1439 );
1440
1441 Ok(())
1442 }
1443
1444 #[cfg(target_arch = "wasm32")]
1446 pub fn reload_cache_from_global_storage(&self) {
1447 use crate::storage::vfs_sync::{with_global_metadata, with_global_storage};
1448
1449 log::info!("[RELOAD] Starting cache reload for: {}", self.db_name);
1450
1451 #[cfg(target_arch = "wasm32")]
1452 web_sys::console::log_1(
1453 &format!("[RELOAD] BlockStorage.db_name = {}", self.db_name).into(),
1454 );
1455
1456 let fresh_cache = with_global_storage(|storage_map| {
1457 if let Some(db_storage) = storage_map.borrow().get(&self.db_name) {
1458 log::info!(
1459 "[RELOAD] Found {} blocks in GLOBAL_STORAGE",
1460 db_storage.len()
1461 );
1462 db_storage.clone()
1463 } else {
1464 log::warn!(
1465 "[RELOAD] No blocks found in GLOBAL_STORAGE for {}",
1466 self.db_name
1467 );
1468 std::collections::HashMap::new()
1469 }
1470 });
1471
1472 let block_count = fresh_cache.len();
1473 log::info!(
1474 "[RELOAD] Loading {} blocks into cache and marking as dirty",
1475 block_count
1476 );
1477
1478 with_global_metadata(|metadata_map| {
1481 if let Some(db_metadata) = metadata_map.borrow().get(&self.db_name) {
1482 log::info!("[RELOAD] Found {} metadata entries", db_metadata.len());
1483 self.checksum_manager.clear_checksums();
1485
1486 let mut new_checksums = std::collections::HashMap::new();
1488 let mut new_algos = std::collections::HashMap::new();
1489 for (block_id, meta) in db_metadata {
1490 new_checksums.insert(*block_id, meta.checksum);
1491 new_algos.insert(*block_id, meta.algo);
1492 }
1493 self.checksum_manager.replace_all(new_checksums, new_algos);
1494 } else {
1495 log::warn!("[RELOAD] No metadata found for {}", self.db_name);
1496 self.checksum_manager.clear_checksums();
1498 }
1499 });
1500
1501 let old_lru = {
1503 let mut lru = lock_mutex!(self.lru_order);
1504 std::mem::take(&mut *lru)
1505 };
1506 lock_mutex!(self.cache).clear();
1507
1508 for (block_id, block_data) in fresh_cache {
1510 lock_mutex!(self.cache).insert(block_id, block_data);
1511 }
1512
1513 log::info!(
1514 "[RELOAD] Cache reloaded with {} blocks",
1515 lock_mutex!(self.cache).len()
1516 );
1517
1518 for block_id in old_lru {
1520 if lock_mutex!(self.cache).contains_key(&block_id) {
1521 lock_mutex!(self.lru_order).push_back(block_id);
1522 }
1523 }
1524
1525 let block_ids: Vec<u64> = lock_mutex!(self.cache).keys().copied().collect();
1527 for block_id in block_ids {
1528 if !lock_mutex!(self.lru_order).contains(&block_id) {
1529 lock_mutex!(self.lru_order).push_back(block_id);
1530 }
1531 }
1532
1533 log::info!(
1534 "[RELOAD] Reload complete: {} blocks in cache, {} dirty",
1535 lock_mutex!(self.cache).len(),
1536 lock_mutex!(self.dirty_blocks).len()
1537 );
1538
1539 #[cfg(target_arch = "wasm32")]
1541 {
1542 if let Some(block_0) = lock_mutex!(self.cache).get(&0) {
1543 let is_valid = block_0.len() >= 16 && &block_0[0..16] == b"SQLite format 3\0";
1544 web_sys::console::log_1(
1545 &format!(
1546 "[RELOAD] Block 0 in cache: {} bytes, valid SQLite header: {}",
1547 block_0.len(),
1548 is_valid
1549 )
1550 .into(),
1551 );
1552 if block_0.len() >= 16 {
1553 web_sys::console::log_1(
1554 &format!("[RELOAD] Block 0 header [0..16]: {:02x?}", &block_0[0..16])
1555 .into(),
1556 );
1557 }
1558
1559 if block_0.len() >= 28 {
1561 let db_size_bytes = [block_0[24], block_0[25], block_0[26], block_0[27]];
1562 let db_size_pages = u32::from_be_bytes(db_size_bytes);
1563 web_sys::console::log_1(
1564 &format!(
1565 "[RELOAD] Database size field (offset 24-27): {} pages ({:02x?})",
1566 db_size_pages, db_size_bytes
1567 )
1568 .into(),
1569 );
1570 }
1571
1572 if block_0.len() >= 100 {
1574 web_sys::console::log_1(
1575 &format!(
1576 "[RELOAD] Full header dump [16-31]: {:02x?}",
1577 &block_0[16..32]
1578 )
1579 .into(),
1580 );
1581 web_sys::console::log_1(
1582 &format!(
1583 "[RELOAD] Full header dump [32-47]: {:02x?}",
1584 &block_0[32..48]
1585 )
1586 .into(),
1587 );
1588 web_sys::console::log_1(
1589 &format!(
1590 "[RELOAD] Full header dump [48-63]: {:02x?}",
1591 &block_0[48..64]
1592 )
1593 .into(),
1594 );
1595 web_sys::console::log_1(
1596 &format!(
1597 "[RELOAD] Full header dump [64-79]: {:02x?}",
1598 &block_0[64..80]
1599 )
1600 .into(),
1601 );
1602 web_sys::console::log_1(
1603 &format!(
1604 "[RELOAD] Full header dump [80-99]: {:02x?}",
1605 &block_0[80..100]
1606 )
1607 .into(),
1608 );
1609 }
1610 } else {
1611 web_sys::console::log_1(
1612 &"[RELOAD] ERROR: Block 0 NOT in cache after reload!"
1613 .to_string()
1614 .into(),
1615 );
1616 }
1617 }
1618 }
1619
1620 pub fn get_cache_size(&self) -> usize {
1621 lock_mutex!(self.cache).len()
1622 }
1623
1624 pub fn get_dirty_count(&self) -> usize {
1625 lock_mutex!(self.dirty_blocks).len()
1626 }
1627
1628 pub fn get_db_name(&self) -> &str {
1629 &self.db_name
1630 }
1631
1632 pub fn is_cached(&self, block_id: u64) -> bool {
1633 lock_mutex!(self.cache).contains_key(&block_id)
1634 }
1635
1636 pub async fn allocate_block(&mut self) -> Result<u64, DatabaseError> {
1638 super::allocation::allocate_block_impl(self).await
1639 }
1640
1641 pub async fn deallocate_block(&mut self, block_id: u64) -> Result<(), DatabaseError> {
1643 super::allocation::deallocate_block_impl(self, block_id).await
1644 }
1645
1646 pub fn get_allocated_count(&self) -> usize {
1648 lock_mutex!(self.allocated_blocks).len()
1649 }
1650
1651 #[cfg(target_arch = "wasm32")]
1655 pub async fn crash_simulation_sync(
1656 &mut self,
1657 blocks_written: bool,
1658 ) -> Result<(), DatabaseError> {
1659 log::info!(
1660 "CRASH SIMULATION: Starting crash simulation with blocks_written={}",
1661 blocks_written
1662 );
1663
1664 if blocks_written {
1665 let dirty_blocks = {
1670 let dirty = lock_mutex!(self.dirty_blocks);
1671 dirty.clone()
1672 };
1673
1674 if !dirty_blocks.is_empty() {
1675 log::info!(
1676 "CRASH SIMULATION: Writing {} blocks to IndexedDB before crash",
1677 dirty_blocks.len()
1678 );
1679
1680 let next_commit = self.get_commit_marker() + 1;
1682 let metadata_to_persist: Vec<(u64, BlockMetadataPersist)> = dirty_blocks
1683 .iter()
1684 .map(|(&block_id, block_data)| {
1685 let checksum =
1686 self.checksum_manager
1687 .get_checksum(block_id)
1688 .unwrap_or_else(|| {
1689 ChecksumManager::compute_checksum_with(
1690 block_data,
1691 self.checksum_manager.get_algorithm(block_id),
1692 )
1693 });
1694 (
1695 block_id,
1696 BlockMetadataPersist {
1697 checksum,
1698 last_modified_ms: BlockStorage::now_millis(),
1699 version: next_commit as u32,
1700 algo: self.checksum_manager.get_algorithm(block_id),
1701 },
1702 )
1703 })
1704 .collect();
1705
1706 log::debug!(
1707 "CRASH SIMULATION: About to call persist_to_indexeddb for {} blocks",
1708 dirty_blocks.len()
1709 );
1710
1711 super::wasm_indexeddb::persist_to_indexeddb(
1713 &self.db_name,
1714 dirty_blocks,
1715 metadata_to_persist,
1716 )
1717 .await?;
1718
1719 log::info!("CRASH SIMULATION: persist_to_indexeddb completed successfully");
1720 log::info!(
1721 "CRASH SIMULATION: Blocks written to IndexedDB, simulating crash before commit marker advance"
1722 );
1723
1724 lock_mutex!(self.dirty_blocks).clear();
1726
1727 Ok(())
1731 } else {
1732 log::info!("CRASH SIMULATION: No dirty blocks to write");
1733 Ok(())
1734 }
1735 } else {
1736 log::info!("CRASH SIMULATION: Simulating crash before blocks are written to IndexedDB");
1738
1739 Ok(())
1741 }
1742 }
1743
1744 #[cfg(target_arch = "wasm32")]
1747 pub async fn crash_simulation_partial_sync(
1748 &mut self,
1749 blocks_to_write: &[u64],
1750 ) -> Result<(), DatabaseError> {
1751 log::info!(
1752 "CRASH SIMULATION: Starting partial crash simulation for {} blocks",
1753 blocks_to_write.len()
1754 );
1755
1756 let dirty_blocks = {
1757 let dirty = lock_mutex!(self.dirty_blocks);
1758 dirty.clone()
1759 };
1760
1761 let partial_blocks: std::collections::HashMap<u64, Vec<u8>> = dirty_blocks
1763 .into_iter()
1764 .filter(|(block_id, _)| blocks_to_write.contains(block_id))
1765 .collect();
1766
1767 if !partial_blocks.is_empty() {
1768 log::info!(
1769 "CRASH SIMULATION: Writing {} out of {} blocks before crash",
1770 partial_blocks.len(),
1771 blocks_to_write.len()
1772 );
1773
1774 let next_commit = self.get_commit_marker() + 1;
1775 let metadata_to_persist: Vec<(u64, BlockMetadataPersist)> = partial_blocks
1776 .iter()
1777 .map(|(&block_id, block_data)| {
1778 let checksum =
1779 self.checksum_manager
1780 .get_checksum(block_id)
1781 .unwrap_or_else(|| {
1782 ChecksumManager::compute_checksum_with(
1783 block_data,
1784 self.checksum_manager.get_algorithm(block_id),
1785 )
1786 });
1787 (
1788 block_id,
1789 BlockMetadataPersist {
1790 checksum,
1791 last_modified_ms: BlockStorage::now_millis(),
1792 version: next_commit as u32,
1793 algo: self.checksum_manager.get_algorithm(block_id),
1794 },
1795 )
1796 })
1797 .collect();
1798
1799 super::wasm_indexeddb::persist_to_indexeddb(
1801 &self.db_name,
1802 partial_blocks.clone(),
1803 metadata_to_persist,
1804 )
1805 .await?;
1806
1807 {
1809 let mut dirty = lock_mutex!(self.dirty_blocks);
1810 for block_id in partial_blocks.keys() {
1811 dirty.remove(block_id);
1812 }
1813 }
1814
1815 log::info!(
1816 "CRASH SIMULATION: Partial blocks written, simulating crash before commit marker advance"
1817 );
1818
1819 }
1821
1822 Ok(())
1823 }
1824
1825 #[cfg(target_arch = "wasm32")]
1829 pub async fn perform_crash_recovery(&mut self) -> Result<CrashRecoveryAction, DatabaseError> {
1830 log::info!(
1831 "CRASH RECOVERY: Starting crash recovery scan for database: {}",
1832 self.db_name
1833 );
1834
1835 let current_marker = self.get_commit_marker();
1837 log::info!("CRASH RECOVERY: Current commit marker: {}", current_marker);
1838
1839 let inconsistent_blocks = self.scan_for_inconsistent_blocks(current_marker).await?;
1842
1843 if inconsistent_blocks.is_empty() {
1844 log::info!("CRASH RECOVERY: No inconsistent blocks found, system is consistent");
1845 return Ok(CrashRecoveryAction::NoActionNeeded);
1846 }
1847
1848 log::info!(
1849 "CRASH RECOVERY: Found {} inconsistent blocks that need recovery",
1850 inconsistent_blocks.len()
1851 );
1852
1853 let recovery_action = self.determine_recovery_action(&inconsistent_blocks).await?;
1855
1856 match recovery_action {
1857 CrashRecoveryAction::Rollback => {
1858 log::info!("CRASH RECOVERY: Performing rollback of incomplete transaction");
1859 self.rollback_incomplete_transaction(&inconsistent_blocks)
1860 .await?;
1861 }
1862 CrashRecoveryAction::Finalize => {
1863 log::info!("CRASH RECOVERY: Performing finalization of complete transaction");
1864 self.finalize_complete_transaction(&inconsistent_blocks)
1865 .await?;
1866 }
1867 CrashRecoveryAction::NoActionNeeded => {
1868 }
1870 }
1871
1872 log::info!("CRASH RECOVERY: Recovery completed successfully");
1873 Ok(recovery_action)
1874 }
1875
1876 #[cfg(target_arch = "wasm32")]
1878 async fn scan_for_inconsistent_blocks(
1879 &self,
1880 commit_marker: u64,
1881 ) -> Result<Vec<(u64, u64)>, DatabaseError> {
1882 log::info!(
1883 "CRASH RECOVERY: Scanning for blocks with version > {}",
1884 commit_marker
1885 );
1886
1887 let mut inconsistent_blocks = Vec::new();
1890
1891 vfs_sync::with_global_metadata(|meta_map| {
1892 if let Some(db_meta) = meta_map.borrow().get(&self.db_name) {
1893 for (block_id, metadata) in db_meta.iter() {
1894 if metadata.version as u64 > commit_marker {
1895 log::info!(
1896 "CRASH RECOVERY: Found inconsistent block {} with version {} > marker {}",
1897 block_id,
1898 metadata.version,
1899 commit_marker
1900 );
1901 inconsistent_blocks.push((*block_id, metadata.version as u64));
1902 }
1903 }
1904 }
1905 });
1906
1907 Ok(inconsistent_blocks)
1908 }
1909
1910 #[cfg(target_arch = "wasm32")]
1912 async fn determine_recovery_action(
1913 &self,
1914 inconsistent_blocks: &[(u64, u64)],
1915 ) -> Result<CrashRecoveryAction, DatabaseError> {
1916 let expected_next_commit = self.get_commit_marker() + 1;
1921 let all_same_version = inconsistent_blocks
1922 .iter()
1923 .all(|(_, version)| *version == expected_next_commit);
1924
1925 if all_same_version && !inconsistent_blocks.is_empty() {
1926 log::info!(
1927 "CRASH RECOVERY: All inconsistent blocks have expected version {}, finalizing transaction",
1928 expected_next_commit
1929 );
1930 Ok(CrashRecoveryAction::Finalize)
1931 } else {
1932 log::info!(
1933 "CRASH RECOVERY: Inconsistent block versions detected, rolling back transaction"
1934 );
1935 Ok(CrashRecoveryAction::Rollback)
1936 }
1937 }
1938
1939 #[cfg(target_arch = "wasm32")]
1941 async fn rollback_incomplete_transaction(
1942 &mut self,
1943 inconsistent_blocks: &[(u64, u64)],
1944 ) -> Result<(), DatabaseError> {
1945 log::info!(
1946 "CRASH RECOVERY: Rolling back {} inconsistent blocks",
1947 inconsistent_blocks.len()
1948 );
1949
1950 vfs_sync::with_global_metadata(|meta_map| {
1952 if let Some(db_meta) = meta_map.borrow_mut().get_mut(&self.db_name) {
1953 for (block_id, _) in inconsistent_blocks {
1954 log::info!(
1955 "CRASH RECOVERY: Removing inconsistent block {} from metadata",
1956 block_id
1957 );
1958 db_meta.remove(block_id);
1959 }
1960 }
1961 });
1962
1963 vfs_sync::with_global_storage(|storage_map| {
1965 if let Some(db_storage) = storage_map.borrow_mut().get_mut(&self.db_name) {
1966 for (block_id, _) in inconsistent_blocks {
1967 log::info!(
1968 "CRASH RECOVERY: Removing inconsistent block {} from global storage",
1969 block_id
1970 );
1971 db_storage.remove(block_id);
1972 }
1973 }
1974 });
1975
1976 for (block_id, _) in inconsistent_blocks {
1978 lock_mutex!(self.cache).remove(block_id);
1979 lock_mutex!(self.lru_order).retain(|&id| id != *block_id);
1981 }
1982
1983 let block_ids_to_delete: Vec<u64> = inconsistent_blocks.iter().map(|(id, _)| *id).collect();
1985 if !block_ids_to_delete.is_empty() {
1986 log::info!(
1987 "CRASH RECOVERY: Deleting {} blocks from IndexedDB",
1988 block_ids_to_delete.len()
1989 );
1990 super::wasm_indexeddb::delete_blocks_from_indexeddb(
1991 &self.db_name,
1992 &block_ids_to_delete,
1993 )
1994 .await?;
1995 log::info!("CRASH RECOVERY: Successfully deleted blocks from IndexedDB");
1996 }
1997
1998 log::info!("CRASH RECOVERY: Rollback completed");
1999 Ok(())
2000 }
2001
2002 #[cfg(target_arch = "wasm32")]
2004 async fn finalize_complete_transaction(
2005 &mut self,
2006 inconsistent_blocks: &[(u64, u64)],
2007 ) -> Result<(), DatabaseError> {
2008 log::info!(
2009 "CRASH RECOVERY: Finalizing transaction for {} blocks",
2010 inconsistent_blocks.len()
2011 );
2012
2013 if let Some((_, target_version)) = inconsistent_blocks.first() {
2015 let new_commit_marker = *target_version;
2016
2017 vfs_sync::with_global_commit_marker(|cm| {
2019 cm.borrow_mut()
2020 .insert(self.db_name.clone(), new_commit_marker);
2021 });
2022
2023 log::info!(
2024 "CRASH RECOVERY: Advanced commit marker from {} to {}",
2025 self.get_commit_marker(),
2026 new_commit_marker
2027 );
2028
2029 for (block_id, _) in inconsistent_blocks {
2031 if let Ok(data) = self.read_block_sync(*block_id) {
2033 self.checksum_manager.store_checksum(*block_id, &data);
2034 log::info!(
2035 "CRASH RECOVERY: Updated checksum for finalized block {}",
2036 block_id
2037 );
2038 }
2039 }
2040 }
2041
2042 log::info!("CRASH RECOVERY: Finalization completed");
2043 Ok(())
2044 }
2045
2046 #[cfg(target_arch = "wasm32")]
2050 pub async fn start_leader_election(&self) -> Result<(), DatabaseError> {
2051 if self.leader_election.borrow().is_none() {
2052 log::debug!(
2053 "BlockStorage::start_leader_election() - Creating new LeaderElectionManager for {}",
2054 self.db_name
2055 );
2056 let mut manager =
2057 super::leader_election::LeaderElectionManager::new(self.db_name.clone());
2058 log::debug!("BlockStorage::start_leader_election() - Calling manager.start_election()");
2059 manager.start_election_sync()?;
2060 log::debug!(
2061 "BlockStorage::start_leader_election() - Election started, storing manager"
2062 );
2063 *self.leader_election.borrow_mut() = Some(manager);
2064 } else {
2065 log::debug!(
2067 "BlockStorage::start_leader_election() - Election already exists, forcing leadership"
2068 );
2069 if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2070 manager.try_become_leader_sync(true)?;
2071 }
2072 }
2073 Ok(())
2074 }
2075
2076 #[cfg(target_arch = "wasm32")]
2079 pub fn was_leader(&self) -> bool {
2080 self.leader_election
2081 .borrow()
2082 .as_ref()
2083 .map(|m| m.state.borrow().is_leader)
2084 .unwrap_or(false)
2085 }
2086
2087 #[cfg(target_arch = "wasm32")]
2091 pub fn stop_heartbeat_sync(&self) {
2092 if let Ok(mut manager_ref) = self.leader_election.try_borrow_mut() {
2094 if let Some(ref mut manager) = *manager_ref {
2095 if let Some(interval_id) = manager.heartbeat_interval.take() {
2097 if let Some(window) = web_sys::window() {
2098 window.clear_interval_with_handle(interval_id);
2099 web_sys::console::log_1(
2100 &format!(
2101 "[DROP] Cleared heartbeat interval {} for {}",
2102 interval_id, self.db_name
2103 )
2104 .into(),
2105 );
2106 }
2107 }
2108 }
2111 } else {
2112 web_sys::console::log_1(
2114 &format!(
2115 "[DROP] Skipping heartbeat stop for {} (already handled)",
2116 self.db_name
2117 )
2118 .into(),
2119 );
2120 }
2121 }
2122
2123 #[cfg(target_arch = "wasm32")]
2125 pub async fn is_leader(&self) -> bool {
2126 if self.leader_election.borrow().is_none() {
2128 log::debug!(
2129 "BlockStorage::is_leader() - Starting leader election for {}",
2130 self.db_name
2131 );
2132 if let Err(e) = self.start_leader_election().await {
2133 log::error!("Failed to start leader election: {:?}", e);
2134 return false;
2135 }
2136 log::debug!("BlockStorage::is_leader() - Leader election started successfully");
2137 } else {
2138 log::debug!(
2139 "BlockStorage::is_leader() - Leader election already exists for {}",
2140 self.db_name
2141 );
2142 }
2143
2144 if let Some(ref mut manager) = *self.leader_election.borrow_mut() {
2145 let is_leader = manager.is_leader_sync();
2146
2147 if !is_leader {
2149 let has_no_leader = manager.state.borrow().leader_id.is_none();
2150 if has_no_leader {
2151 log::debug!(
2152 "No current leader for {} - triggering re-election",
2153 self.db_name
2154 );
2155 if let Err(error) = manager.try_become_leader_sync(false) {
2156 log::error!("Failed to re-elect leader for {}: {}", self.db_name, error);
2157 return false;
2158 }
2159
2160 let new_is_leader = manager.state.borrow().is_leader;
2162 if new_is_leader && manager.heartbeat_interval.is_none() {
2163 if let Err(error) = manager.start_heartbeat() {
2164 log::error!(
2165 "Failed to start heartbeat for {}: {}",
2166 self.db_name,
2167 error
2168 );
2169 return false;
2170 }
2171 }
2172
2173 log::debug!(
2174 "is_leader() for {} = {} (after re-election)",
2175 self.db_name,
2176 new_is_leader
2177 );
2178 return new_is_leader;
2179 }
2180 }
2181
2182 log::debug!("is_leader() for {} = {}", self.db_name, is_leader);
2183 is_leader
2184 } else {
2185 log::debug!("No leader election manager for {}", self.db_name);
2186 false
2187 }
2188 }
2189
2190 #[cfg(target_arch = "wasm32")]
2192 pub async fn stop_leader_election(&self) -> Result<(), DatabaseError> {
2193 let manager = self.leader_election.borrow_mut().take();
2194 if let Some(mut manager) = manager {
2195 manager.stop_election_sync()?;
2196 }
2197 Ok(())
2198 }
2199
2200 #[cfg(target_arch = "wasm32")]
2202 pub async fn send_leader_heartbeat(&self) -> Result<(), DatabaseError> {
2203 if let Some(ref manager) = *self.leader_election.borrow() {
2204 manager.send_heartbeat_sync()
2205 } else {
2206 Err(DatabaseError::new(
2207 "LEADER_ELECTION_ERROR",
2208 "Leader election not started",
2209 ))
2210 }
2211 }
2212
2213 #[cfg(target_arch = "wasm32")]
2215 pub async fn get_last_leader_heartbeat(&self) -> Result<u64, DatabaseError> {
2216 if let Some(ref manager) = *self.leader_election.borrow() {
2217 Ok(manager.get_last_heartbeat_sync())
2218 } else {
2219 Err(DatabaseError::new(
2220 "LEADER_ELECTION_ERROR",
2221 "Leader election not started",
2222 ))
2223 }
2224 }
2225
2226 pub fn get_metrics(&self) -> super::observability::StorageMetrics {
2230 let dirty_count = self.get_dirty_count();
2231 let dirty_bytes = dirty_count * BLOCK_SIZE;
2232
2233 #[cfg(not(target_arch = "wasm32"))]
2234 let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2235 (
2236 self.sync_count.load(Ordering::SeqCst),
2237 self.timer_sync_count.load(Ordering::SeqCst),
2238 self.debounce_sync_count.load(Ordering::SeqCst),
2239 self.last_sync_duration_ms.load(Ordering::SeqCst),
2240 )
2241 };
2242
2243 #[cfg(target_arch = "wasm32")]
2244 let (sync_count, timer_sync_count, debounce_sync_count, last_sync_duration_ms) = {
2245 (self.observability.get_sync_count(), 0, 0, 1)
2247 };
2248
2249 let error_count = self.observability.get_error_count();
2250 let checksum_failures = self.observability.get_checksum_failures();
2251
2252 let total_operations = sync_count + error_count;
2254 let (throughput_blocks_per_sec, throughput_bytes_per_sec) = self
2255 .observability
2256 .calculate_throughput(last_sync_duration_ms);
2257 let error_rate = self.observability.calculate_error_rate(total_operations);
2258
2259 super::observability::StorageMetrics {
2260 dirty_count,
2261 dirty_bytes,
2262 sync_count,
2263 timer_sync_count,
2264 debounce_sync_count,
2265 error_count,
2266 checksum_failures,
2267 last_sync_duration_ms,
2268 throughput_blocks_per_sec,
2269 throughput_bytes_per_sec,
2270 error_rate,
2271 }
2272 }
2273
2274 #[cfg(not(target_arch = "wasm32"))]
2276 pub fn set_sync_callbacks(
2277 &mut self,
2278 on_sync_start: super::observability::SyncStartCallback,
2279 on_sync_success: super::observability::SyncSuccessCallback,
2280 on_sync_failure: super::observability::SyncFailureCallback,
2281 ) {
2282 self.observability.sync_start_callback = Some(on_sync_start);
2283 self.observability.sync_success_callback = Some(on_sync_success);
2284 self.observability.sync_failure_callback = Some(on_sync_failure);
2285 }
2286
2287 #[cfg(not(target_arch = "wasm32"))]
2289 pub fn set_backpressure_callback(
2290 &mut self,
2291 callback: super::observability::BackpressureCallback,
2292 ) {
2293 self.observability.backpressure_callback = Some(callback);
2294 }
2295
2296 #[cfg(not(target_arch = "wasm32"))]
2298 pub fn set_error_callback(&mut self, callback: super::observability::ErrorCallback) {
2299 self.observability.error_callback = Some(callback);
2300 }
2301
2302 #[cfg(target_arch = "wasm32")]
2304 pub fn set_sync_success_callback(
2305 &mut self,
2306 callback: super::observability::WasmSyncSuccessCallback,
2307 ) {
2308 self.observability.wasm_sync_success_callback = Some(callback);
2309 }
2310
2311 pub fn is_auto_sync_enabled(&self) -> bool {
2313 lock_mutex!(self.auto_sync_interval).is_some()
2314 }
2315
2316 pub fn get_sync_policy(&self) -> Option<super::SyncPolicy> {
2318 lock_mutex!(self.policy).clone()
2319 }
2320
2321 pub async fn force_sync(&mut self) -> Result<(), DatabaseError> {
2327 log::info!("force_sync: Starting forced synchronization with durability guarantees");
2328
2329 let dirty_count = self.get_dirty_count();
2330 if dirty_count == 0 {
2331 log::debug!("force_sync: No dirty blocks to sync");
2332 return Ok(());
2333 }
2334
2335 log::info!(
2336 "force_sync: Syncing {} dirty blocks with durability guarantee",
2337 dirty_count
2338 );
2339
2340 self.sync().await?;
2342
2343 log::info!("force_sync: Successfully completed forced synchronization");
2344 Ok(())
2345 }
2346
2347 #[cfg(all(feature = "telemetry", not(target_arch = "wasm32")))]
2349 pub fn set_metrics(&mut self, metrics: Option<crate::telemetry::Metrics>) {
2350 self.metrics = metrics;
2351 }
2352
2353 #[cfg(all(feature = "telemetry", target_arch = "wasm32"))]
2354 pub fn set_metrics(&self, metrics: Option<crate::telemetry::Metrics>) {
2355 *self.metrics.borrow_mut() = metrics;
2356 }
2357
2358 #[cfg(all(feature = "telemetry", not(target_arch = "wasm32")))]
2360 pub fn metrics(&self) -> Option<&crate::telemetry::Metrics> {
2361 self.metrics.as_ref()
2362 }
2363
2364 #[cfg(all(feature = "telemetry", target_arch = "wasm32"))]
2365 pub fn metrics(&self) -> Option<crate::telemetry::Metrics> {
2366 self.metrics.borrow().clone()
2367 }
2368
2369 #[cfg(feature = "telemetry")]
2370 pub(crate) fn metrics_snapshot(&self) -> Option<crate::telemetry::Metrics> {
2371 #[cfg(target_arch = "wasm32")]
2372 {
2373 self.metrics()
2374 }
2375 #[cfg(not(target_arch = "wasm32"))]
2376 {
2377 self.metrics.clone()
2378 }
2379 }
2380
2381 #[cfg(feature = "telemetry")]
2383 pub fn new_for_test() -> Self {
2384 #[cfg(target_arch = "wasm32")]
2385 use std::cell::RefCell as Mutex;
2386 Self {
2387 cache: Mutex::new(HashMap::new()),
2388 #[cfg(not(target_arch = "wasm32"))]
2389 dirty_blocks: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2390 #[cfg(target_arch = "wasm32")]
2391 dirty_blocks: std::rc::Rc::new(RefCell::new(HashMap::new())),
2392 allocated_blocks: Mutex::new(HashSet::new()),
2393 deallocated_blocks: Mutex::new(HashSet::new()),
2394 next_block_id: AtomicU64::new(1),
2395 capacity: 128,
2396 lru_order: Mutex::new(VecDeque::new()),
2397 checksum_manager: crate::storage::metadata::ChecksumManager::new(
2398 crate::storage::metadata::ChecksumAlgorithm::FastHash,
2399 ),
2400 #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
2401 base_dir: std::path::PathBuf::from("/tmp/test"),
2402 db_name: "test.db".to_string(),
2403 backend: StorageBackend::IndexedDb,
2404 auto_sync_interval: Mutex::new(None),
2405 #[cfg(not(target_arch = "wasm32"))]
2406 last_auto_sync: std::time::Instant::now(),
2407 policy: Mutex::new(None),
2408 #[cfg(not(target_arch = "wasm32"))]
2409 auto_sync_stop: None,
2410 #[cfg(not(target_arch = "wasm32"))]
2411 auto_sync_thread: None,
2412 #[cfg(not(target_arch = "wasm32"))]
2413 debounce_thread: None,
2414 #[cfg(not(target_arch = "wasm32"))]
2415 tokio_timer_task: None,
2416 #[cfg(not(target_arch = "wasm32"))]
2417 tokio_debounce_task: None,
2418 #[cfg(not(target_arch = "wasm32"))]
2419 last_write_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2420 #[cfg(not(target_arch = "wasm32"))]
2421 threshold_hit: Arc::new(std::sync::atomic::AtomicBool::new(false)),
2422 #[cfg(not(target_arch = "wasm32"))]
2423 sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2424 #[cfg(not(target_arch = "wasm32"))]
2425 timer_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2426 #[cfg(not(target_arch = "wasm32"))]
2427 debounce_sync_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2428 #[cfg(not(target_arch = "wasm32"))]
2429 last_sync_duration_ms: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2430 #[cfg(not(target_arch = "wasm32"))]
2431 sync_sender: None,
2432 #[cfg(not(target_arch = "wasm32"))]
2433 sync_receiver: None,
2434 recovery_report: RecoveryReport::default(),
2435 #[cfg(target_arch = "wasm32")]
2436 leader_election: std::cell::RefCell::new(None),
2437 observability: super::observability::ObservabilityManager::new(),
2438 metrics: Default::default(),
2439 }
2440 }
2441}
2442
2443#[cfg(all(test, target_arch = "wasm32"))]
2444mod wasm_commit_marker_tests {
2445 use super::*;
2446 use wasm_bindgen_test::*;
2447
2448 wasm_bindgen_test_configure!(run_in_browser);
2449
2450 fn set_commit_marker(db: &str, v: u64) {
2452 let db = crate::utils::normalize_db_name(db);
2454 super::vfs_sync::with_global_commit_marker(|cm| {
2455 cm.borrow_mut().insert(db, v);
2456 });
2457 }
2458
2459 fn get_commit_marker(db: &str) -> u64 {
2461 let db = crate::utils::normalize_db_name(db);
2463 vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2464 }
2465
2466 #[wasm_bindgen_test]
2467 async fn gating_returns_zeroed_until_marker_catches_up_wasm() {
2468 let db = "cm_gating_wasm";
2469 let mut s = BlockStorage::new(db).await.expect("create storage");
2470
2471 let bid = s.allocate_block().await.expect("alloc block");
2473 let data_v1 = vec![0x33u8; BLOCK_SIZE];
2474 s.write_block(bid, data_v1.clone()).await.expect("write v1");
2475
2476 s.clear_cache();
2478 let out0 = s.read_block(bid).await.expect("read before commit");
2479 assert_eq!(
2480 out0,
2481 vec![0u8; BLOCK_SIZE],
2482 "uncommitted data must read as zeroed"
2483 );
2484
2485 s.sync().await.expect("sync v1");
2487 s.clear_cache();
2488 let out1 = s.read_block(bid).await.expect("read after commit");
2489 assert_eq!(out1, data_v1, "committed data should be visible");
2490 }
2491
2492 #[wasm_bindgen_test]
2493 async fn invisible_blocks_skip_checksum_verification_wasm() {
2494 let db = "cm_checksum_skip_wasm";
2495 let mut s = BlockStorage::new(db).await.expect("create storage");
2496
2497 let bid = s.allocate_block().await.expect("alloc block");
2498 let data = vec![0x44u8; BLOCK_SIZE];
2499 s.write_block(bid, data).await.expect("write v1");
2500 s.sync().await.expect("sync v1"); set_commit_marker(db, 0);
2504
2505 s.set_block_checksum_for_testing(bid, 1234567);
2507 s.clear_cache();
2508 let out = s
2509 .read_block(bid)
2510 .await
2511 .expect("read while invisible should not error");
2512 assert_eq!(
2513 out,
2514 vec![0u8; BLOCK_SIZE],
2515 "invisible block reads as zeroed"
2516 );
2517
2518 set_commit_marker(db, 1);
2520 s.clear_cache();
2521 let err = s
2522 .read_block(bid)
2523 .await
2524 .expect_err("expected checksum mismatch once visible");
2525 assert_eq!(err.code, "CHECKSUM_MISMATCH");
2526 }
2527
2528 #[wasm_bindgen_test]
2529 async fn commit_marker_advances_and_versions_track_syncs_wasm() {
2530 let db = "cm_versions_wasm";
2531 let mut s = BlockStorage::new_with_capacity(db, 8)
2532 .await
2533 .expect("create storage");
2534
2535 let b1 = s.allocate_block().await.expect("alloc b1");
2536 let b2 = s.allocate_block().await.expect("alloc b2");
2537
2538 s.write_block(b1, vec![1u8; BLOCK_SIZE])
2539 .await
2540 .expect("write b1 v1");
2541 s.write_block(b2, vec![2u8; BLOCK_SIZE])
2542 .await
2543 .expect("write b2 v1");
2544 s.sync().await.expect("sync #1");
2545
2546 let cm1 = get_commit_marker(db);
2547 assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2548 let meta1 = s.get_block_metadata_for_testing();
2549 assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2550 assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2551
2552 s.write_block(b1, vec![3u8; BLOCK_SIZE])
2554 .await
2555 .expect("write b1 v2");
2556 s.sync().await.expect("sync #2");
2557
2558 let cm2 = get_commit_marker(db);
2559 assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2560 let meta2 = s.get_block_metadata_for_testing();
2561 assert_eq!(
2562 meta2.get(&b1).unwrap().1 as u64,
2563 cm2,
2564 "updated block tracks new version"
2565 );
2566 assert_eq!(
2567 meta2.get(&b2).unwrap().1 as u64,
2568 1,
2569 "unchanged block retains prior version"
2570 );
2571 }
2572}
2573
2574#[cfg(all(test, not(target_arch = "wasm32"), not(feature = "fs_persist")))]
2575mod commit_marker_tests {
2576 use super::*;
2577
2578 fn set_commit_marker(db: &str, v: u64) {
2580 let db = crate::utils::normalize_db_name(db);
2582 super::vfs_sync::with_global_commit_marker(|cm| {
2583 cm.borrow_mut().insert(db, v);
2584 });
2585 }
2586
2587 fn get_commit_marker(db: &str) -> u64 {
2589 let db = crate::utils::normalize_db_name(db);
2591 super::vfs_sync::with_global_commit_marker(|cm| cm.borrow().get(&db).copied().unwrap_or(0))
2592 }
2593
2594 #[tokio::test(flavor = "current_thread")]
2595 async fn gating_returns_zeroed_until_marker_catches_up() {
2596 let db = "cm_gating_basic";
2597 println!("DEBUG: Creating BlockStorage for {}", db);
2598 let mut s = BlockStorage::new(db).await.expect("create storage");
2599 println!("DEBUG: BlockStorage created successfully");
2600
2601 let bid = s.allocate_block().await.expect("alloc block");
2603 println!("DEBUG: Allocated block {}", bid);
2604 let data_v1 = vec![0x11u8; BLOCK_SIZE];
2605 s.write_block(bid, data_v1.clone()).await.expect("write v1");
2606 println!("DEBUG: Wrote block {} with data", bid);
2607
2608 s.clear_cache();
2610 let out0 = s.read_block(bid).await.expect("read before commit");
2611 assert_eq!(
2612 out0,
2613 vec![0u8; BLOCK_SIZE],
2614 "uncommitted data must read as zeroed"
2615 );
2616 println!("DEBUG: Pre-sync read returned zeroed data as expected");
2617
2618 println!("DEBUG: About to call sync");
2620 s.sync().await.expect("sync v1");
2621 println!("DEBUG: Sync completed successfully");
2622
2623 let commit_marker = get_commit_marker(db);
2625 println!("DEBUG: Commit marker after sync: {}", commit_marker);
2626
2627 s.clear_cache();
2628 let out1 = s.read_block(bid).await.expect("read after commit");
2629
2630 println!("DEBUG: Expected data: {:?}", &data_v1[..8]);
2632 println!("DEBUG: Actual data: {:?}", &out1[..8]);
2633 println!(
2634 "DEBUG: Data lengths - expected: {}, actual: {}",
2635 data_v1.len(),
2636 out1.len()
2637 );
2638
2639 let data_matches = out1 == data_v1;
2641 println!("DEBUG: Data matches: {}", data_matches);
2642
2643 if !data_matches {
2644 println!("DEBUG: Data mismatch detected - investigating further");
2645 let is_all_zeros = out1.iter().all(|&b| b == 0);
2647 println!("DEBUG: Is all zeros: {}", is_all_zeros);
2648
2649 println!("DEBUG: Final commit marker: {}", get_commit_marker(db));
2651
2652 panic!("Data mismatch: expected committed data to be visible after sync");
2653 }
2654
2655 println!("DEBUG: Test passed - data is visible after commit");
2656 }
2657
2658 #[tokio::test(flavor = "current_thread")]
2659 async fn invisible_blocks_skip_checksum_verification() {
2660 let db = "cm_checksum_skip";
2661 let mut s = BlockStorage::new(db).await.expect("create storage");
2662
2663 let bid = s.allocate_block().await.expect("alloc block");
2664 let data = vec![0xAAu8; BLOCK_SIZE];
2665 s.write_block(bid, data.clone()).await.expect("write v1");
2666 s.sync().await.expect("sync v1"); set_commit_marker(db, 0);
2670
2671 s.set_block_checksum_for_testing(bid, 1234567);
2673 s.clear_cache();
2674 let out = s
2675 .read_block(bid)
2676 .await
2677 .expect("read while invisible should not error");
2678 assert_eq!(
2679 out,
2680 vec![0u8; BLOCK_SIZE],
2681 "invisible block reads as zeroed"
2682 );
2683
2684 set_commit_marker(db, 1);
2686 s.clear_cache();
2687 let err = s
2688 .read_block(bid)
2689 .await
2690 .expect_err("expected checksum mismatch once visible");
2691 assert_eq!(err.code, "CHECKSUM_MISMATCH");
2692 }
2693
2694 #[tokio::test(flavor = "current_thread")]
2695 async fn commit_marker_advances_and_versions_track_syncs() {
2696 let db = "cm_versions";
2697 let mut s = BlockStorage::new_with_capacity(db, 8)
2698 .await
2699 .expect("create storage");
2700
2701 let b1 = s.allocate_block().await.expect("alloc b1");
2702 let b2 = s.allocate_block().await.expect("alloc b2");
2703
2704 s.write_block(b1, vec![1u8; BLOCK_SIZE])
2705 .await
2706 .expect("write b1 v1");
2707 s.write_block(b2, vec![2u8; BLOCK_SIZE])
2708 .await
2709 .expect("write b2 v1");
2710 s.sync().await.expect("sync #1");
2711
2712 let cm1 = get_commit_marker(db);
2713 assert_eq!(cm1, 1, "first sync should advance commit marker to 1");
2714 let meta1 = s.get_block_metadata_for_testing();
2715 assert_eq!(meta1.get(&b1).unwrap().1 as u64, cm1);
2716 assert_eq!(meta1.get(&b2).unwrap().1 as u64, cm1);
2717
2718 s.write_block(b1, vec![3u8; BLOCK_SIZE])
2720 .await
2721 .expect("write b1 v2");
2722 s.sync().await.expect("sync #2");
2723
2724 let cm2 = get_commit_marker(db);
2725 assert_eq!(cm2, 2, "second sync should advance commit marker to 2");
2726 let meta2 = s.get_block_metadata_for_testing();
2727 assert_eq!(
2728 meta2.get(&b1).unwrap().1 as u64,
2729 cm2,
2730 "updated block tracks new version"
2731 );
2732 assert_eq!(
2733 meta2.get(&b2).unwrap().1 as u64,
2734 1,
2735 "unchanged block retains prior version"
2736 );
2737 }
2738}