Skip to main content

absurder_sql/storage/
sync_operations.rs

1//! Sync operations for BlockStorage
2//! This module contains the core sync implementation logic
3
4// Reentrancy-safe lock macros
5#[cfg(target_arch = "wasm32")]
6macro_rules! lock_mutex {
7    ($mutex:expr) => {
8        $mutex
9            .try_borrow_mut()
10            .expect("RefCell borrow failed - reentrancy detected in sync_operations.rs")
11    };
12}
13
14#[cfg(not(target_arch = "wasm32"))]
15macro_rules! lock_mutex {
16    ($mutex:expr) => {
17        $mutex.lock()
18    };
19}
20
21#[allow(unused_macros)]
22#[cfg(target_arch = "wasm32")]
23macro_rules! try_lock_mutex {
24    ($mutex:expr) => {
25        $mutex
26    };
27}
28
29#[allow(unused_macros)]
30#[cfg(not(target_arch = "wasm32"))]
31macro_rules! try_lock_mutex {
32    ($mutex:expr) => {
33        $mutex.lock()
34    };
35}
36
37use super::block_storage::BlockStorage;
38use crate::types::DatabaseError;
39
40#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
41use std::collections::HashMap;
42#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
43use std::sync::atomic::Ordering;
44
45#[cfg(any(
46    target_arch = "wasm32",
47    all(not(target_arch = "wasm32"), not(feature = "fs_persist"))
48))]
49use super::metadata::BlockMetadataPersist;
50#[cfg(any(
51    target_arch = "wasm32",
52    all(not(target_arch = "wasm32"), not(feature = "fs_persist"))
53))]
54use super::vfs_sync;
55
56#[cfg(target_arch = "wasm32")]
57use std::collections::HashMap;
58
59#[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
60use super::block_storage::GLOBAL_METADATA_TEST;
61
62/// Internal sync implementation shared by sync() and sync_now()
63pub fn sync_implementation_impl(storage: &mut BlockStorage) -> Result<(), DatabaseError> {
64    #[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
65    let start = std::time::Instant::now();
66
67    // Record sync start for observability
68    let dirty_count = lock_mutex!(storage.dirty_blocks).len();
69    let dirty_bytes = dirty_count * super::block_storage::BLOCK_SIZE;
70    storage
71        .observability
72        .record_sync_start(dirty_count, dirty_bytes);
73
74    // Invoke sync start callback if set
75    #[cfg(not(target_arch = "wasm32"))]
76    if let Some(ref callback) = storage.observability.sync_start_callback {
77        callback(dirty_count, dirty_bytes);
78    }
79
80    // Call the existing fs_persist implementation for native builds
81    #[cfg(all(not(target_arch = "wasm32"), feature = "fs_persist"))]
82    {
83        storage.fs_persist_sync()
84    }
85
86    // For native non-fs_persist builds, use simple in-memory sync with commit marker handling
87    #[cfg(all(not(target_arch = "wasm32"), not(feature = "fs_persist")))]
88    {
89        let current_dirty = lock_mutex!(storage.dirty_blocks).len();
90        log::info!(
91            "Syncing {} dirty blocks (native non-fs_persist)",
92            current_dirty
93        );
94
95        let to_persist: Vec<(u64, Vec<u8>)> = {
96            let dirty = lock_mutex!(storage.dirty_blocks);
97            dirty.iter().map(|(k, v)| (*k, v.clone())).collect()
98        };
99        let ids: Vec<u64> = to_persist.iter().map(|(k, _)| *k).collect();
100        let blocks_synced = ids.len(); // Capture length before moving ids
101
102        // Determine next commit version for native path
103        let next_commit: u64 = vfs_sync::with_global_commit_marker(|cm| {
104            let cm = cm.borrow();
105            let current = cm.get(&storage.db_name).copied().unwrap_or(0);
106            current + 1
107        });
108
109        // Store blocks in global storage with versioning
110        vfs_sync::with_global_storage(|gs| {
111            let mut storage_map = gs.borrow_mut();
112            let db_storage = storage_map
113                .entry(storage.db_name.clone())
114                .or_insert_with(HashMap::new);
115            for (block_id, data) in to_persist {
116                db_storage.insert(block_id, data);
117            }
118        });
119
120        // Store metadata with per-commit versioning
121        GLOBAL_METADATA_TEST.with(|meta| {
122            let mut meta_map = meta.lock();
123            let db_meta = meta_map
124                .entry(storage.db_name.clone())
125                .or_insert_with(HashMap::new);
126            for block_id in ids {
127                if let Some(checksum) = storage.checksum_manager.get_checksum(block_id) {
128                    // Use the per-commit version so entries remain invisible until the commit marker advances
129                    let version = next_commit as u32;
130                    db_meta.insert(
131                        block_id,
132                        BlockMetadataPersist {
133                            checksum,
134                            last_modified_ms: std::time::SystemTime::now()
135                                .duration_since(std::time::UNIX_EPOCH)
136                                .unwrap_or_default()
137                                .as_millis() as u64,
138                            version,
139                            algo: storage.checksum_manager.get_algorithm(block_id),
140                        },
141                    );
142                }
143            }
144        });
145
146        // Atomically advance the commit marker after all data and metadata are persisted
147        vfs_sync::with_global_commit_marker(|cm| {
148            let mut cm_map = cm.borrow_mut();
149            cm_map.insert(storage.db_name.clone(), next_commit);
150        });
151
152        // Clear dirty blocks
153        {
154            let mut dirty = lock_mutex!(storage.dirty_blocks);
155            dirty.clear();
156        }
157
158        // Update sync metrics
159        storage.sync_count.fetch_add(1, Ordering::SeqCst);
160        let elapsed = start.elapsed();
161        let ms = elapsed.as_millis() as u64;
162        let ms = if ms == 0 { 1 } else { ms };
163        storage.last_sync_duration_ms.store(ms, Ordering::SeqCst);
164
165        // Record sync success for observability
166        storage.observability.record_sync_success(ms, blocks_synced);
167
168        // Invoke sync success callback if set
169        if let Some(ref callback) = storage.observability.sync_success_callback {
170            callback(ms, blocks_synced);
171        }
172
173        storage.evict_if_needed();
174        return Ok(());
175    }
176
177    #[cfg(target_arch = "wasm32")]
178    {
179        // WASM implementation
180        let current_dirty = lock_mutex!(storage.dirty_blocks).len();
181        log::info!("Syncing {} dirty blocks (WASM)", current_dirty);
182
183        // For WASM, persist dirty blocks to global storage
184        let to_persist: Vec<(u64, Vec<u8>)> = {
185            let dirty = lock_mutex!(storage.dirty_blocks);
186            dirty.iter().map(|(k, v)| (*k, v.clone())).collect()
187        };
188        let ids: Vec<u64> = to_persist.iter().map(|(k, _)| *k).collect();
189        // Determine next commit version so that all metadata written in this sync share the same version
190        let next_commit: u64 = vfs_sync::with_global_commit_marker(|cm| {
191            let cm = cm;
192            let current = cm.borrow().get(&storage.db_name).copied().unwrap_or(0);
193            current + 1
194        });
195        vfs_sync::with_global_storage(|gs| {
196            let mut storage_map = gs.borrow_mut();
197            let db_storage = storage_map
198                .entry(storage.db_name.clone())
199                .or_insert_with(HashMap::new);
200            for (block_id, data) in &to_persist {
201                // Check if block already exists in global storage with committed data
202                let should_update = if let Some(existing) = db_storage.get(block_id) {
203                    if existing != data {
204                        // Check if existing data has committed metadata (version > 0)
205                        let has_committed_metadata = vfs_sync::with_global_metadata(|meta| {
206                            if let Some(db_meta) = meta.borrow().get(&storage.db_name) {
207                                if let Some(metadata) = db_meta.get(block_id) {
208                                    metadata.version > 0
209                                } else {
210                                    false
211                                }
212                            } else {
213                                false
214                            }
215                        });
216
217                        if has_committed_metadata {
218                            // CRITICAL FIX: Never overwrite committed data to prevent corruption
219                            false // Never overwrite committed data
220                        } else {
221                            true // Update uncommitted data
222                        }
223                    } else {
224                        true // Same data, safe to update
225                    }
226                } else {
227                    true // No existing data, safe to insert
228                };
229
230                if should_update {
231                    db_storage.insert(*block_id, data.clone());
232                }
233            }
234        });
235        // Persist corresponding metadata entries
236        vfs_sync::with_global_metadata(|meta| {
237            let mut meta_guard = meta.borrow_mut();
238            let db_meta = meta_guard
239                .entry(storage.db_name.clone())
240                .or_insert_with(HashMap::new);
241            for block_id in ids {
242                if let Some(checksum) = storage.checksum_manager.get_checksum(block_id) {
243                    // Use the per-commit version so entries remain invisible until the commit marker advances
244                    let version = next_commit as u32;
245                    db_meta.insert(
246                        block_id,
247                        BlockMetadataPersist {
248                            checksum,
249                            last_modified_ms: BlockStorage::now_millis(),
250                            version,
251                            algo: storage.checksum_manager.get_algorithm(block_id),
252                        },
253                    );
254                }
255            }
256        });
257        // Atomically advance the commit marker after all data and metadata are persisted
258        vfs_sync::with_global_commit_marker(|cm| {
259            let cm_map = cm;
260            cm_map
261                .borrow_mut()
262                .insert(storage.db_name.clone(), next_commit);
263        });
264
265        // Spawn async IndexedDB persistence (fire and forget for sync compatibility)
266        let db_name = storage.db_name.clone();
267        wasm_bindgen_futures::spawn_local(async move {
268            use wasm_bindgen::JsCast;
269
270            // Get IndexedDB factory (works in both Window and Worker contexts)
271            let global = js_sys::global();
272            let indexed_db_value = match js_sys::Reflect::get(
273                &global,
274                &wasm_bindgen::JsValue::from_str("indexedDB"),
275            ) {
276                Ok(val) => val,
277                Err(_) => {
278                    log::error!("IndexedDB property access failed - cannot persist");
279                    return;
280                }
281            };
282
283            if indexed_db_value.is_null() || indexed_db_value.is_undefined() {
284                log::warn!(
285                    "IndexedDB unavailable for sync (private browsing?) - data not persisted to IndexedDB"
286                );
287                return;
288            }
289
290            let idb_factory = match indexed_db_value.dyn_into::<web_sys::IdbFactory>() {
291                Ok(factory) => factory,
292                Err(_) => {
293                    log::error!("IndexedDB property is not an IdbFactory - cannot persist");
294                    return;
295                }
296            };
297
298            let open_req = match idb_factory.open_with_u32("block_storage", 2) {
299                Ok(req) => req,
300                Err(e) => {
301                    log::error!("Failed to open IndexedDB for sync: {:?}", e);
302                    return;
303                }
304            };
305
306            // Set up upgrade handler to create object stores if needed.
307            // Typed web-sys Closure instead of js_sys::Function::new_no_args so it
308            // does not require 'unsafe-eval' (blocked by MV3 extension CSP).
309            let upgrade_callback =
310                wasm_bindgen::closure::Closure::wrap(Box::new(move |event: web_sys::Event| {
311                    if let Some(target) = event.target() {
312                        let request: web_sys::IdbOpenDbRequest = target.unchecked_into();
313                        if let Ok(result) = request.result() {
314                            let db: web_sys::IdbDatabase = result.unchecked_into();
315                            let names = db.object_store_names();
316                            if !names.contains("blocks") {
317                                let _ = db.create_object_store("blocks");
318                            }
319                            if !names.contains("metadata") {
320                                let _ = db.create_object_store("metadata");
321                            }
322                        }
323                    }
324                }) as Box<dyn FnMut(_)>);
325            open_req.set_onupgradeneeded(Some(upgrade_callback.as_ref().unchecked_ref()));
326            upgrade_callback.forget();
327
328            // Use event-based approach for opening database
329            let (tx, rx) = futures::channel::oneshot::channel();
330            let tx = std::rc::Rc::new(std::cell::RefCell::new(Some(tx)));
331
332            let success_tx = tx.clone();
333            let success_callback =
334                wasm_bindgen::closure::Closure::wrap(Box::new(move |event: web_sys::Event| {
335                    if let Some(tx) = success_tx.borrow_mut().take() {
336                        let target = event.target().unwrap();
337                        let request: web_sys::IdbOpenDbRequest = target.unchecked_into();
338                        let result = request.result().unwrap();
339                        let _ = tx.send(Ok(result));
340                    }
341                }) as Box<dyn FnMut(_)>);
342
343            let error_tx = tx.clone();
344            let error_callback =
345                wasm_bindgen::closure::Closure::wrap(Box::new(move |event: web_sys::Event| {
346                    if let Some(tx) = error_tx.borrow_mut().take() {
347                        let _ = tx.send(Err(format!("IndexedDB open failed: {:?}", event)));
348                    }
349                }) as Box<dyn FnMut(_)>);
350
351            open_req.set_onsuccess(Some(success_callback.as_ref().unchecked_ref()));
352            open_req.set_onerror(Some(error_callback.as_ref().unchecked_ref()));
353
354            let db_result = rx.await;
355
356            // Keep closures alive
357            success_callback.forget();
358            error_callback.forget();
359
360            match db_result {
361                Ok(Ok(db_value)) => {
362                    if let Ok(db) = db_value.dyn_into::<web_sys::IdbDatabase>() {
363                        // Start transaction for both blocks and metadata
364                        let store_names = js_sys::Array::new();
365                        store_names.push(&wasm_bindgen::JsValue::from_str("blocks"));
366                        store_names.push(&wasm_bindgen::JsValue::from_str("metadata"));
367
368                        let transaction = db
369                            .transaction_with_str_sequence_and_mode(
370                                &store_names,
371                                web_sys::IdbTransactionMode::Readwrite,
372                            )
373                            .unwrap();
374
375                        let blocks_store = transaction.object_store("blocks").unwrap();
376                        let metadata_store = transaction.object_store("metadata").unwrap();
377
378                        // Persist all blocks
379                        // IMPORTANT: Use COLON format to match wasm_indexeddb.rs and restore logic
380                        for (block_id, data) in &to_persist {
381                            let key = wasm_bindgen::JsValue::from_str(&format!(
382                                "{}:{}",
383                                db_name, block_id
384                            ));
385                            let value = js_sys::Uint8Array::from(&data[..]);
386                            blocks_store.put_with_key(&value, &key).unwrap();
387                        }
388
389                        // Persist commit marker
390                        // IMPORTANT: Use COLON format to match wasm_indexeddb.rs and restore logic
391                        let commit_key =
392                            wasm_bindgen::JsValue::from_str(&format!("{}:commit_marker", db_name));
393                        let commit_value = wasm_bindgen::JsValue::from_f64(next_commit as f64);
394                        metadata_store
395                            .put_with_key(&commit_value, &commit_key)
396                            .unwrap();
397
398                        // Use event-based approach for transaction completion
399                        let (tx_tx, tx_rx) = futures::channel::oneshot::channel();
400                        let tx_tx = std::rc::Rc::new(std::cell::RefCell::new(Some(tx_tx)));
401
402                        let tx_complete_tx = tx_tx.clone();
403                        let tx_complete_callback = wasm_bindgen::closure::Closure::wrap(Box::new(
404                            move |_event: web_sys::Event| {
405                                if let Some(tx) = tx_complete_tx.borrow_mut().take() {
406                                    let _ = tx.send(Ok(()));
407                                }
408                            },
409                        )
410                            as Box<dyn FnMut(_)>);
411
412                        let tx_error_tx = tx_tx.clone();
413                        let tx_error_callback = wasm_bindgen::closure::Closure::wrap(Box::new(
414                            move |event: web_sys::Event| {
415                                if let Some(tx) = tx_error_tx.borrow_mut().take() {
416                                    let _ =
417                                        tx.send(Err(format!("Transaction failed: {:?}", event)));
418                                }
419                            },
420                        )
421                            as Box<dyn FnMut(_)>);
422
423                        transaction
424                            .set_oncomplete(Some(tx_complete_callback.as_ref().unchecked_ref()));
425                        transaction.set_onerror(Some(tx_error_callback.as_ref().unchecked_ref()));
426
427                        let _ = tx_rx.await;
428
429                        // Keep closures alive
430                        tx_complete_callback.forget();
431                        tx_error_callback.forget();
432                    }
433                }
434                _ => {} // Silently ignore errors in background persistence
435            }
436        });
437        // Clear dirty blocks after successful persistence
438        {
439            let mut dirty = lock_mutex!(storage.dirty_blocks);
440            dirty.clear();
441        }
442
443        // Record sync success for observability (WASM)
444        // For WASM, we don't have precise timing, so use a default duration
445        storage.observability.record_sync_success(1, current_dirty);
446
447        // Invoke WASM sync success callback if set
448        #[cfg(target_arch = "wasm32")]
449        if let Some(ref callback) = storage.observability.wasm_sync_success_callback {
450            callback(1, current_dirty);
451        }
452
453        storage.evict_if_needed();
454        Ok(())
455    }
456}