nodedb-lite 0.1.0

Embedded edge client for NodeDB — offline vector search, graph traversal, and CRDT sync
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

// SPDX-License-Identifier: Apache-2.0

//! Sparse cold recovery remains independent of search field declarations.

use crate::engine::{
    document::history::ops::current_document_page,
    fts::rebuild::{PAGE_BYTES, PAGE_RECORDS},
};
use crate::{
    error::LiteError,
    nodedb::{convert::loro_value_to_document, core::types::NodeDbLite, lock_ext::LockExt},
    storage::engine::StorageEngine,
};

impl<S: StorageEngine> NodeDbLite<S> {
    pub(super) async fn rebuild_sparse_documents(&self) -> Result<(), LiteError> {
        let collections = self.crdt.lock_or_recover().collection_names();
        for collection in collections {
            if collection.starts_with("__")
                || self.is_authoritative_text_collection(&collection).await?
            {
                continue;
            }
            let mut cursor: Option<String> = None;
            loop {
                let ids = self.crdt.lock_or_recover().live_ids_page(
                    &collection,
                    cursor.as_deref(),
                    PAGE_RECORDS,
                    PAGE_BYTES,
                )?;
                if ids.is_empty() {
                    break;
                }
                for id in ids {
                    let fields = self
                        .crdt
                        .lock_or_recover()
                        .read(&collection, &id)
                        .map(|value| loro_value_to_document(&id, &value).fields);
                    if let Some(fields) = fields {
                        self.sparse_state
                            .manager
                            .lock_or_recover()
                            .index_document_fields(&collection, &id, &fields);
                    }
                    cursor = Some(id);
                }
            }
        }
        for collection in self.list_bitemporal_collections().await? {
            let mut cursor: Option<Vec<u8>> = None;
            loop {
                let page = current_document_page(
                    &*self.storage,
                    &collection,
                    cursor.as_deref(),
                    PAGE_RECORDS,
                    PAGE_BYTES,
                )
                .await?;
                let Some(last_key) = page.last_key else {
                    break;
                };
                for (_, id, fields) in page.entries {
                    self.sparse_state
                        .manager
                        .lock_or_recover()
                        .index_document_fields(&collection, &id, &fields);
                }
                cursor = Some(last_key);
            }
        }
        Ok(())
    }
}