nodedb-fts 0.6.1

Shared full-text search engine for NodeDB — inverted index, BM25 scoring, analyzers, fuzzy matching
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

// SPDX-License-Identifier: Apache-2.0

//! Fieldnorm storage: SmallFloat-encoded document lengths per collection.
//!
//! Stores a compact `Vec<u8>` array indexed by surrogate. Each byte is
//! a SmallFloat-encoded document length. Persisted as metadata blob via
//! the backend's `read_meta`/`write_meta`.

use nodedb_types::Surrogate;

use crate::backend::FtsBackend;
use crate::codec::smallfloat;
use crate::index::FtsIndex;
use crate::scope::IndexScope;

impl<B: FtsBackend> FtsIndex<B> {
    /// Get the fieldnorm (SmallFloat-encoded doc length) for a doc.
    ///
    /// Returns the decoded approximate u32 length, or `None` if not stored.
    pub fn read_fieldnorm<'a>(
        &self,
        database_id: u64,
        tid: u64,
        index: impl Into<IndexScope<'a>>,
        doc_id: Surrogate,
    ) -> Result<Option<u32>, B::Error> {
        let data = self
            .backend
            .read_meta(database_id, tid, index.into(), "fieldnorms")?;
        match data {
            Some(bytes) if (doc_id.0 as usize) < bytes.len() => {
                Ok(Some(smallfloat::decode(bytes[doc_id.0 as usize])))
            }
            _ => Ok(None),
        }
    }

    /// Write a fieldnorm byte for a surrogate. Grows the array if needed.
    pub fn write_fieldnorm<'a>(
        &self,
        database_id: u64,
        tid: u64,
        index: impl Into<IndexScope<'a>>,
        doc_id: Surrogate,
        doc_length: u32,
    ) -> Result<(), B::Error> {
        let index = index.into();
        let mut data = self
            .backend
            .read_meta(database_id, tid, index, "fieldnorms")?
            .unwrap_or_default();

        let idx = doc_id.0 as usize;
        if idx >= data.len() {
            data.resize(idx + 1, 0);
        }
        data[idx] = smallfloat::encode(doc_length);

        self.backend
            .write_meta(database_id, tid, index, "fieldnorms", &data)
    }
}

#[cfg(test)]
mod tests {
    use nodedb_types::Surrogate;

    use crate::backend::memory::MemoryBackend;
    use crate::codec::smallfloat;
    use crate::index::FtsIndex;
    use crate::test_support::test_governor;

    const DB: u64 = 0;
    const T: u64 = 1;

    #[test]
    fn fieldnorm_roundtrip() {
        let idx = FtsIndex::new(MemoryBackend::new(), test_governor());
        idx.write_fieldnorm(DB, T, "col", Surrogate(0), 100)
            .unwrap();
        idx.write_fieldnorm(DB, T, "col", Surrogate(5), 50).unwrap();

        let norm0 = idx
            .read_fieldnorm(DB, T, "col", Surrogate(0))
            .unwrap()
            .unwrap();
        let norm5 = idx
            .read_fieldnorm(DB, T, "col", Surrogate(5))
            .unwrap()
            .unwrap();

        assert!(norm0 <= 100);
        assert!(norm5 <= 50);
        assert_eq!(norm0, smallfloat::decode(smallfloat::encode(100)));
        assert_eq!(norm5, smallfloat::decode(smallfloat::encode(50)));
    }

    #[test]
    fn fieldnorm_missing_doc() {
        let idx = FtsIndex::new(MemoryBackend::new(), test_governor());
        assert_eq!(
            idx.read_fieldnorm(DB, T, "col", Surrogate(99)).unwrap(),
            None
        );
    }

    #[test]
    fn fieldnorm_overwrite() {
        let idx = FtsIndex::new(MemoryBackend::new(), test_governor());
        idx.write_fieldnorm(DB, T, "col", Surrogate(0), 100)
            .unwrap();
        idx.write_fieldnorm(DB, T, "col", Surrogate(0), 200)
            .unwrap();

        let norm = idx
            .read_fieldnorm(DB, T, "col", Surrogate(0))
            .unwrap()
            .unwrap();
        assert_eq!(norm, smallfloat::decode(smallfloat::encode(200)));
    }
}