leviathan/
fields.rs

1//! Field paths into JSON records, and the compiled [`Mapping`] that turns a
2//! record into what the index stores.
3
4use std::collections::HashSet;
5
6use anyhow::Result;
7use serde_json::Value;
8
9use crate::config::Config;
10use crate::text::{facet_token, group_token, normalize_date};
11
12/// A dotted path such as `asset.id` or `steps[].text`. Arrays anywhere on the
13/// path are flattened, so `[]` is optional. A key that itself contains dots
14/// (common in CSV headers) is matched literally before the path is split.
15#[derive(Debug, Clone)]
16pub struct FieldPath {
17    pub raw: String,
18    segs: Vec<String>,
19    rest: Vec<String>,
20}
21
22impl FieldPath {
23    pub fn parse(raw: &str) -> Self {
24        let segs: Vec<String> = raw
25            .split('.')
26            .map(|s| s.trim().trim_end_matches("[]").to_string())
27            .filter(|s| !s.is_empty())
28            .collect();
29        let rest = (0..segs.len()).map(|i| segs[i..].join(".")).collect();
30        Self { raw: raw.trim().to_string(), segs, rest }
31    }
32
33    /// Every value at this path, arrays flattened.
34    pub fn values<'a>(&self, record: &'a Value) -> Vec<&'a Value> {
35        let mut out = Vec::new();
36        self.walk(record, 0, &mut out);
37        out
38    }
39
40    fn walk<'a>(&self, value: &'a Value, i: usize, out: &mut Vec<&'a Value>) {
41        match value {
42            Value::Array(items) => items.iter().for_each(|item| self.walk(item, i, out)),
43            _ if i == self.segs.len() => out.push(value),
44            Value::Object(map) => {
45                if self.segs.len() - i > 1
46                    && let Some(child) = map.get(&self.rest[i])
47                {
48                    return self.walk(child, self.segs.len(), out);
49                }
50                if let Some(child) = map.get(&self.segs[i]) {
51                    self.walk(child, i + 1, out);
52                }
53            }
54            _ => {}
55        }
56    }
57}
58
59/// Scalars as text. Objects contribute their nested strings.
60fn push_text(value: &Value, out: &mut Vec<String>, top: bool) {
61    match value {
62        Value::String(s) => out.push(s.clone()),
63        Value::Number(n) if top => out.push(n.to_string()),
64        Value::Bool(b) if top => out.push(b.to_string()),
65        Value::Array(items) => items.iter().for_each(|v| push_text(v, out, top)),
66        Value::Object(map) => map.values().for_each(|v| push_text(v, out, false)),
67        _ => {}
68    }
69}
70
71/// The config compiled for fast per-record extraction.
72#[derive(Debug, Clone)]
73pub struct Mapping {
74    pub config: Config,
75    id: Option<FieldPath>,
76    title: Vec<FieldPath>,
77    text: Vec<FieldPath>,
78    group: Option<FieldPath>,
79    group_name: Option<FieldPath>,
80    date: Vec<FieldPath>,
81    pub filters: Vec<FieldPath>,
82    pub display: Vec<FieldPath>,
83    empty: HashSet<String>,
84    boosts: Vec<(Vec<FieldPath>, Option<String>, f64)>,
85}
86
87/// What the index stores for one record.
88#[derive(Debug, Clone, Default)]
89pub struct Prepared {
90    pub id: Option<String>,
91    pub title: Option<String>,
92    pub body: String,
93    /// Group key and name, searchable in their own column.
94    pub names: String,
95    pub group: Option<String>,
96    pub group_name: Option<String>,
97    pub date: Option<String>,
98    pub boost: f64,
99    /// `(field, value)` pairs, value as first seen (trimmed).
100    pub facets: Vec<(String, String)>,
101    /// Synthetic FTS tokens for the group and each facet value.
102    pub tags: String,
103}
104
105/// `text` with the record's own group key and name removed (ASCII
106/// case-insensitive). They are indexed once, in their own column; repeated in
107/// a title they would outrank the words that tell records apart.
108fn without_names(text: &str, names: &[&str]) -> String {
109    let mut out = text.to_string();
110    for name in names.iter().filter(|n| n.chars().count() >= 3) {
111        if out.is_ascii() && name.is_ascii() {
112            let lower = out.to_ascii_lowercase();
113            let needle = name.to_ascii_lowercase();
114            let mut kept = String::with_capacity(out.len());
115            let mut last = 0;
116            for (i, _) in lower.match_indices(&needle) {
117                kept.push_str(&out[last..i]);
118                kept.push(' ');
119                last = i + needle.len();
120            }
121            kept.push_str(&out[last..]);
122            out = kept;
123        } else {
124            out = out.replace(name, " ");
125        }
126    }
127    out
128}
129
130fn placeholder_key(s: &str) -> String {
131    s.trim().trim_end_matches(['.', '!']).trim().to_lowercase()
132}
133
134impl Mapping {
135    pub fn new(config: &Config) -> Result<Self> {
136        config.validate()?;
137        let f = &config.fields;
138        let paths = |v: &[String]| v.iter().map(|p| FieldPath::parse(p)).collect::<Vec<_>>();
139        Ok(Self {
140            config: config.clone(),
141            id: f.id.as_deref().map(FieldPath::parse),
142            title: paths(&f.title),
143            text: paths(&f.text),
144            group: f.group.as_deref().map(FieldPath::parse),
145            group_name: f.group_name.as_deref().map(FieldPath::parse),
146            date: paths(&f.date),
147            filters: paths(&f.filters),
148            display: paths(&f.display),
149            empty: f.empty_values.iter().map(|s| placeholder_key(s)).collect(),
150            boosts: config
151                .rank
152                .boost
153                .iter()
154                .map(|b| (paths(&b.field), b.equals.as_ref().map(|e| e.trim().to_lowercase()), b.weight))
155                .collect(),
156        })
157    }
158
159    pub fn has_id(&self) -> bool {
160        self.id.is_some()
161    }
162    pub fn has_group(&self) -> bool {
163        self.group.is_some()
164    }
165    pub fn has_date(&self) -> bool {
166        !self.date.is_empty()
167    }
168    pub fn title_paths(&self) -> &[FieldPath] {
169        &self.title
170    }
171    pub fn text_paths(&self) -> &[FieldPath] {
172        &self.text
173    }
174
175    /// Missing, blank, or a configured placeholder.
176    pub fn is_empty_value(&self, s: &str) -> bool {
177        let key = placeholder_key(s);
178        key.is_empty() || self.empty.contains(&key)
179    }
180
181    /// Present text values at `path`, placeholders dropped.
182    pub fn strings(&self, path: &FieldPath, record: &Value) -> Vec<String> {
183        let mut out = Vec::new();
184        for v in path.values(record) {
185            push_text(v, &mut out, true);
186        }
187        out.retain(|s| !self.is_empty_value(s));
188        out
189    }
190
191    pub fn first(&self, paths: &[FieldPath], record: &Value) -> Option<String> {
192        paths.iter().find_map(|p| self.strings(p, record).into_iter().next()).map(|s| s.trim().to_string())
193    }
194
195    pub fn record_id(&self, record: &Value) -> Option<String> {
196        self.id.as_ref().and_then(|p| self.first(std::slice::from_ref(p), record))
197    }
198
199    /// The record's date, normalized when recognizable.
200    pub fn date_of(&self, record: &Value) -> Option<String> {
201        self.first(&self.date, record).map(|d| normalize_date(&d).unwrap_or(d))
202    }
203
204    pub fn group_of(&self, record: &Value) -> (Option<String>, Option<String>) {
205        let one =
206            |p: &Option<FieldPath>| p.as_ref().and_then(|p| self.first(std::slice::from_ref(p), record));
207        let group = one(&self.group);
208        let name = if group.is_some() { one(&self.group_name) } else { None };
209        (group, name)
210    }
211
212    pub fn prepare(&self, record: &Value) -> Prepared {
213        let (group, group_name) = self.group_of(record);
214        let own: Vec<&str> = group.iter().chain(&group_name).map(String::as_str).collect();
215        let names = own.join("\n");
216        let title = self.first(&self.title, record).map(|t| without_names(&t, &own));
217        let mut body: Vec<String> = Vec::new();
218        if self.text.is_empty() {
219            push_text(record, &mut body, true);
220            body.retain(|s| !self.is_empty_value(s));
221        } else {
222            for p in &self.text {
223                body.extend(self.strings(p, record));
224            }
225        }
226        for b in &mut body {
227            *b = without_names(b, &own);
228        }
229
230        let date = self.date_of(record);
231
232        let mut facets: Vec<(String, String)> = Vec::new();
233        let mut seen = HashSet::new();
234        for p in &self.filters {
235            for v in self.strings(p, record) {
236                let v: String = v.trim().chars().take(200).collect();
237                if seen.insert((p.raw.clone(), v.to_lowercase())) {
238                    facets.push((p.raw.clone(), v));
239                }
240            }
241        }
242
243        let mut boost = 1.0;
244        for (paths, equals, weight) in &self.boosts {
245            let hit = paths.iter().any(|p| {
246                let values = self.strings(p, record);
247                match equals {
248                    Some(want) => values.iter().any(|v| v.trim().to_lowercase() == *want),
249                    None => !values.is_empty(),
250                }
251            });
252            if hit {
253                boost += weight;
254            }
255        }
256
257        let mut tags: Vec<String> = Vec::with_capacity(facets.len() + 1);
258        tags.extend(group.as_deref().map(group_token));
259        tags.extend(facets.iter().map(|(f, v)| facet_token(f, v)));
260
261        Prepared {
262            id: self.record_id(record),
263            title,
264            body: body.join("\n"),
265            names,
266            group,
267            group_name,
268            date,
269            boost,
270            facets,
271            tags: tags.join(" "),
272        }
273    }
274}
275
276#[cfg(test)]
277mod tests {
278    use super::*;
279    use serde_json::json;
280
281    #[test]
282    fn paths_flatten_arrays_and_match_dotted_keys() {
283        let doc = json!({"a": {"b": [{"c": 1}, {"c": 2}]}, "x.y": "flat", "tags": ["p", "q"]});
284        let vals = |p: &str| FieldPath::parse(p).values(&doc).into_iter().cloned().collect::<Vec<_>>();
285        assert_eq!(vals("a.b[].c"), [json!(1), json!(2)]);
286        assert_eq!(vals("a.b.c"), [json!(1), json!(2)]);
287        assert_eq!(vals("x.y"), [json!("flat")]);
288        assert_eq!(vals("tags[]"), [json!("p"), json!("q")]);
289        assert!(vals("a.missing").is_empty());
290    }
291
292    #[test]
293    fn prepare_applies_placeholders_facets_and_boosts() {
294        let cfg: Config = toml::from_str(
295            r#"
296            [fields]
297            id = "id"
298            title = ["subject", "body"]
299            text = ["subject", "resolution"]
300            group = "customer.id"
301            group_name = "customer.name"
302            date = "closed"
303            filters = ["status", "tags"]
304            empty_values = ["n/a", "done"]
305            [[rank.boost]]
306            field = "resolution"
307            weight = 0.2
308            [[rank.boost]]
309            field = "status"
310            equals = "Closed"
311            weight = 0.05
312            "#,
313        )
314        .unwrap();
315        let m = Mapping::new(&cfg).unwrap();
316        let p = m.prepare(&json!({
317            "id": 7, "subject": "Login fails", "resolution": "Done.",
318            "customer": {"id": "ACME", "name": "Acme Corp"}, "closed": "2024-03-01 10:00",
319            "status": "closed", "tags": ["sso", "SSO", "web"]
320        }));
321        assert_eq!(p.id.as_deref(), Some("7"));
322        assert_eq!(p.title.as_deref(), Some("Login fails"));
323        assert_eq!(p.date.as_deref(), Some("2024-03-01T10:00:00"));
324        assert_eq!(p.group.as_deref(), Some("ACME"));
325        assert!(!p.body.contains("Done"));
326        assert_eq!(p.facets.len(), 3);
327        assert!((p.boost - 1.05).abs() < 1e-9);
328        assert_eq!(p.tags.split(' ').count(), 4);
329    }
330}
leviathan/fields.rs

leviathan/
fields.rs
