second-brain-core 0.13.3

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use serde_json::Value;

use super::sync_node_type_to_str;
use crate::redact::{Finding, redact};
use crate::schema::SyncRecord;

struct TextField {
    name: &'static str,
    /// Whether the node's embedding is computed from this field, so a secret
    /// in it means the stored vector goes too.
    feeds_vector: bool,
}

/// The text fields of each synced node type that ingest redacts.
fn payload_text_fields(node_type: &str) -> &'static [TextField] {
    match node_type {
        "memory" => &[
            TextField {
                name: "content",
                feeds_vector: true,
            },
            TextField {
                name: "context",
                feeds_vector: true,
            },
            TextField {
                name: "retired_reason",
                feeds_vector: false,
            },
        ],
        "entity" => &[
            TextField {
                name: "name",
                feeds_vector: true,
            },
            TextField {
                name: "aliases",
                feeds_vector: false,
            },
        ],
        "relation" => &[TextField {
            name: "context",
            feeds_vector: false,
        }],
        _ => &[],
    }
}

/// The payload's findings and, when there are any, the payload with its text
/// fields redacted and its embedding emptied if a field it was computed from
/// changed. None for a payload that is not JSON.
pub(super) fn scan_payload(node_type: &str, data: &str) -> Option<(Option<String>, Vec<Finding>)> {
    let mut value: Value = serde_json::from_str(data).ok()?;
    let findings = redact_payload(node_type, &mut value);
    if findings.is_empty() {
        return Some((None, findings));
    }
    Some((serde_json::to_string(&value).ok(), findings))
}

/// A record from a peer that has not redacted its own store yet, redacted the
/// way the pass redacts this store's log; None when nothing needed it.
pub(super) fn redacted_record(record: &SyncRecord) -> Option<SyncRecord> {
    let node_type = sync_node_type_to_str(record.node_type);
    let data = record.data.as_ref()?;
    if !payload_has_findings(node_type, data) {
        return None;
    }
    let mut data = data.clone();
    redact_payload(node_type, &mut data);
    Some(SyncRecord {
        data: Some(data),
        ..record.clone()
    })
}

fn payload_has_findings(node_type: &str, value: &Value) -> bool {
    let Some(object) = value.as_object() else {
        return false;
    };
    payload_text_fields(node_type)
        .iter()
        .any(|field| match object.get(field.name) {
            Some(Value::String(text)) => redact(text).changed(),
            Some(Value::Array(items)) => items
                .iter()
                .any(|item| item.as_str().is_some_and(|text| redact(text).changed())),
            _ => false,
        })
}

/// Redacts the payload's text fields in place and empties its embedding when
/// a field the embedding was computed from changed.
fn redact_payload(node_type: &str, value: &mut Value) -> Vec<Finding> {
    let mut findings = Vec::new();
    let Some(object) = value.as_object_mut() else {
        return findings;
    };
    let mut vector_stale = false;
    for field in payload_text_fields(node_type) {
        let changed = match object.get_mut(field.name) {
            Some(Value::String(text)) => redact_in_place(text, &mut findings),
            Some(Value::Array(items)) => items.iter_mut().fold(false, |changed, item| match item {
                Value::String(text) => redact_in_place(text, &mut findings) || changed,
                _ => changed,
            }),
            _ => false,
        };
        vector_stale |= changed && field.feeds_vector;
    }
    if vector_stale && object.contains_key("embedding") {
        object.insert("embedding".to_string(), Value::Array(Vec::new()));
    }
    findings
}

fn redact_in_place(text: &mut String, findings: &mut Vec<Finding>) -> bool {
    let redacted = redact(text);
    if !redacted.changed() {
        return false;
    }
    *text = redacted.text;
    findings.extend(redacted.findings);
    true
}