use crate::memory::identity::{
aliases_value, derive_aliases, identity_props_after_write, same_as_lost, same_as_pairs,
SameAsPair, ALIASES_FIELD, MAX_ALIASES,
};
use crate::memory_schema::{NAME_FIELD, PROVISIONAL_LABEL, PROVISIONAL_PROP};
use crate::{GraphDb, NS_PROP};
use core_storage::fs::Fs;
use core_storage::{GraphError, Result, Value};
use std::collections::BTreeMap;
use std::collections::BTreeSet;
const MIN_TEXT_CHARS: usize = 1;
const MAX_TEXT_CHARS: usize = 4000;
pub const NOTE_KINDS: [&str; 3] = ["note", "decision", "todo"];
pub const DEFAULT_NOTE_KIND: &str = "note";
#[must_use]
pub fn unix_now_secs() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs() as i64)
}
pub fn dedup_keep_order(keys: &mut Vec<String>) {
let mut seen = BTreeSet::new();
keys.retain(|k| seen.insert(k.clone()));
}
const ABOUT_EDGE: &str = "ABOUT";
const DEFAULT_SOURCE: &str = "agent";
const MAX_FULLTEXT_PAIRS: usize = 32;
const MAX_PROVISIONAL_PER_COMMIT: usize = 20;
#[derive(Debug, Clone)]
pub struct EntityIn {
pub key: String,
pub label: String,
pub props: BTreeMap<String, Value>,
pub aliases: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct FactIn {
pub subject: String,
pub predicate: String,
pub object: String,
}
pub struct RememberInput<'a> {
pub text: &'a str,
pub about: &'a [String],
pub kind: &'a str,
pub ts: i64,
pub source: Option<&'a str>,
pub entities: &'a [EntityIn],
pub facts: &'a [FactIn],
}
#[derive(Debug, Default, serde::Serialize)]
pub struct RememberReport {
pub note: String,
pub created: usize,
pub matched: usize,
pub derived: usize,
pub provisional: Vec<String>,
pub provisional_capped: Vec<String>,
pub fulltext_declared: Vec<String>,
pub same_as: Vec<SameAsPair>,
pub same_as_lost: Vec<SameAsPair>,
}
fn refuse_blank(value: &str, argument: impl FnOnce() -> String) -> Result<()> {
if value.trim().is_empty() {
return Err(GraphError::IngestError {
detail: format!(
"{} must not be empty or only whitespace, got {value:?}",
argument()
),
});
}
if crate::digest::sanitize(value).trim().is_empty() {
return Err(GraphError::IngestError {
detail: format!(
"{} must not be only invisible characters, got {value:?}",
argument()
),
});
}
Ok(())
}
pub fn describe_entity<F: Fs>(
db: &mut GraphDb<F>,
key: &str,
label: Option<&str>,
props: &[(String, Value)],
) -> Result<bool> {
describe_entity_with_aliases(db, key, label, props, &[])
}
pub fn describe_entity_with_aliases<F: Fs>(
db: &mut GraphDb<F>,
key: &str,
label: Option<&str>,
props: &[(String, Value)],
aliases: &[String],
) -> Result<bool> {
refuse_blank(key, || "key".to_string())?;
if let Some(label) = label {
refuse_blank(label, || "label".to_string())?;
}
let mut props = props.to_vec();
let identity = identity_props_after_write(db, key, &props, aliases)?;
props.extend(identity);
if db.has_node(key) {
if !props.is_empty() {
db.set_props(key, props)?;
}
let _ = db.remove_prop(key, PROVISIONAL_PROP)?;
Ok(false)
} else {
db.insert_node(label.unwrap_or(PROVISIONAL_LABEL), key, props)?;
let _ = db.remove_prop(key, PROVISIONAL_PROP)?;
Ok(true)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct UpsertOutcome {
pub label: String,
pub created: bool,
pub updated_fields: usize,
pub same_as_lost: Vec<SameAsPair>,
}
pub fn upsert_entity<F: Fs>(
db: &mut GraphDb<F>,
key: &str,
label: Option<&str>,
mut row: BTreeMap<String, Value>,
aliases: &[String],
) -> Result<UpsertOutcome> {
refuse_blank(key, || "key".to_string())?;
row.remove("id");
if db.has_node(key) {
let stored_label = db
.node_ref(key)
.map(|n| n.label().to_string())
.unwrap_or_default();
if let Some(requested) = label {
if requested != stored_label {
return Err(GraphError::IngestError {
detail: format!(
"'{key}' exists as {stored_label:?}, not {requested:?}; this release \
cannot relabel a node. Pass the label in remember's 'entities' when you \
know it (at first mention, before it goes provisional), or use a \
different key."
),
});
}
}
let mut to_set: Vec<(String, Value)> = Vec::new();
for (field, v) in row {
if field == NS_PROP && Some(&v) == db.namespace_of(key).map(Value::Str).as_ref() {
continue;
}
to_set.push((field, v));
}
let updated_fields = to_set.len();
let keys = [key.to_string()];
let same_as_before = same_as_pairs(db, &keys);
describe_entity_with_aliases(db, key, None, &to_set, aliases)?;
let same_as_lost = same_as_lost(&same_as_before, &same_as_pairs(db, &keys));
Ok(UpsertOutcome {
label: stored_label,
created: false,
updated_fields,
same_as_lost,
})
} else {
let Some(label) = label else {
return Err(GraphError::IngestError {
detail: "label required when creating a new entity".into(),
});
};
row.insert("id".to_string(), Value::Str(key.to_string()));
let props: Vec<(String, Value)> = row.into_iter().collect();
describe_entity_with_aliases(db, key, Some(label), &props, aliases)?;
Ok(UpsertOutcome {
label: label.to_string(),
created: true,
updated_fields: 0,
same_as_lost: Vec::new(),
})
}
}
pub fn remember<F: Fs>(db: &mut GraphDb<F>, input: &RememberInput<'_>) -> Result<RememberReport> {
let text = input.text.trim();
let len = text.chars().count();
if !(MIN_TEXT_CHARS..=MAX_TEXT_CHARS).contains(&len) {
return Err(GraphError::IngestError {
detail: format!(
"remember: text must be {MIN_TEXT_CHARS}..={MAX_TEXT_CHARS} characters \
after trimming, got {len}"
),
});
}
if !NOTE_KINDS.contains(&input.kind) {
return Err(GraphError::IngestError {
detail: format!(
"remember: kind must be one of {}, got {:?}",
NOTE_KINDS.join(", "),
input.kind
),
});
}
for (i, k) in input.about.iter().enumerate() {
refuse_blank(k, || format!("remember: about[{i}]"))?;
}
for (i, entity) in input.entities.iter().enumerate() {
refuse_blank(&entity.key, || format!("remember: entities[{i}].key"))?;
refuse_blank(&entity.label, || format!("remember: entities[{i}].label"))?;
}
for (i, fact) in input.facts.iter().enumerate() {
refuse_blank(&fact.subject, || format!("remember: facts[{i}].subject"))?;
refuse_blank(&fact.predicate, || {
format!("remember: facts[{i}].predicate")
})?;
refuse_blank(&fact.object, || format!("remember: facts[{i}].object"))?;
}
let entity_identity: Vec<Vec<(String, Value)>> = input
.entities
.iter()
.map(|e| {
let props: Vec<(String, Value)> = e
.props
.iter()
.map(|(f, v)| (f.clone(), v.clone()))
.collect();
identity_props_after_write(db, &e.key, &props, &e.aliases)
})
.collect::<Result<_>>()?;
let mut fulltext = db.fulltext_pairs();
if !fulltext.contains(&("Note".to_string(), "text".to_string())) {
db.enable_fulltext("Note", "text")?;
fulltext.push(("Note".to_string(), "text".to_string()));
}
let mut fulltext_declared: Vec<String> = Vec::new();
{
let mut declared_this_call: BTreeSet<&str> = BTreeSet::new();
for entity in input.entities {
let label = entity.label.as_str();
if db.node_ref(&entity.key).is_some_and(|n| n.label() != label) {
continue;
}
if !declared_this_call.insert(label) {
continue;
}
let pair = (label.to_string(), NAME_FIELD.to_string());
if fulltext.contains(&pair) || fulltext.len() >= MAX_FULLTEXT_PAIRS {
continue;
}
db.enable_fulltext(label, NAME_FIELD)?;
fulltext.push(pair);
fulltext_declared.push(label.to_string());
}
}
let source = input.source.unwrap_or(DEFAULT_SOURCE).to_string();
let key = note_key(input.ts, text);
let mut about: Vec<String> = Vec::with_capacity(input.about.len());
{
let mut seen: BTreeSet<&str> = BTreeSet::new();
for k in input.about {
if seen.insert(k.as_str()) {
about.push(k.clone());
}
}
}
let entity_keys: BTreeSet<&str> = input.entities.iter().map(|e| e.key.as_str()).collect();
let entity_existed: Vec<bool> = input.entities.iter().map(|e| db.has_node(&e.key)).collect();
let about_existed: Vec<Option<bool>> = about
.iter()
.map(|k| {
if entity_keys.contains(k.as_str()) {
None
} else {
Some(db.has_node(k))
}
})
.collect();
let note_existed = db.has_node(&key);
let (already_about, derived_before): (BTreeSet<String>, usize) = if note_existed {
let edges = db.node_edges(&key)?;
(
edges
.iter()
.filter(|e| e.edge_type == ABOUT_EDGE && e.src_key == key)
.map(|e| e.dst_key.clone())
.collect(),
edges.iter().filter(|e| e.derived).count(),
)
} else {
(BTreeSet::new(), 0)
};
let known: BTreeSet<&str> = entity_keys
.iter()
.copied()
.chain(about.iter().map(String::as_str))
.collect();
let mut fact_endpoints: Vec<&str> = Vec::new();
{
let mut seen: BTreeSet<&str> = BTreeSet::new();
for fact in input.facts {
for k in [fact.subject.as_str(), fact.object.as_str()] {
if !known.contains(k) && seen.insert(k) {
fact_endpoints.push(k);
}
}
}
}
let fact_endpoint_existed: Vec<bool> = fact_endpoints.iter().map(|k| db.has_node(k)).collect();
let entity_key_list: Vec<String> = input.entities.iter().map(|e| e.key.clone()).collect();
let same_as_before = same_as_pairs(db, &entity_key_list);
let mut report = RememberReport {
note: key.clone(),
fulltext_declared,
..Default::default()
};
let mut batch = db.batch();
let mut provisional_count = 0usize;
let mut capped: BTreeSet<String> = BTreeSet::new();
for ((entity, existed), identity) in input
.entities
.iter()
.zip(&entity_existed)
.zip(&entity_identity)
{
if *existed {
for (field, value) in &entity.props {
batch.set_prop(&entity.key, field, value.clone());
}
for (field, value) in identity {
batch.set_prop(&entity.key, field, value.clone());
}
report.matched += 1;
} else {
let mut props: Vec<(String, Value)> = entity
.props
.iter()
.map(|(f, v)| (f.clone(), v.clone()))
.collect();
props.extend(identity.iter().cloned());
batch.insert_node(&entity.label, &entity.key, props);
report.created += 1;
}
batch.remove_prop(&entity.key, PROVISIONAL_PROP);
}
for (k, status) in about.iter().zip(&about_existed) {
match status {
None => {}
Some(true) => report.matched += 1,
Some(false) => {
if provisional_count < MAX_PROVISIONAL_PER_COMMIT {
batch.insert_node(PROVISIONAL_LABEL, k, stub_props(k));
report.provisional.push(k.clone());
provisional_count += 1;
} else {
report.provisional_capped.push(k.clone());
capped.insert(k.clone());
}
}
}
}
if !note_existed {
let mut props: Vec<(String, Value)> = vec![
("id".into(), Value::Str(key.clone())),
("text".into(), Value::Str(text.to_string())),
("kind".into(), Value::Str(input.kind.to_string())),
("ts".into(), Value::Int(input.ts)),
("source".into(), Value::Str(source)),
];
if !about.is_empty() {
props.push((
"about".into(),
Value::List(about.iter().cloned().map(Value::Str).collect()),
));
}
batch.insert_node("Note", &key, props);
}
for k in &about {
if capped.contains(k) || already_about.contains(k) {
continue;
}
batch.insert_edge(ABOUT_EDGE, &key, k);
}
for (k, existed) in fact_endpoints.iter().zip(&fact_endpoint_existed) {
if *existed {
report.matched += 1;
continue;
}
if provisional_count < MAX_PROVISIONAL_PER_COMMIT {
batch.insert_node(PROVISIONAL_LABEL, k, stub_props(k));
report.provisional.push((*k).to_string());
provisional_count += 1;
} else {
report.provisional_capped.push((*k).to_string());
capped.insert((*k).to_string());
}
}
for fact in input.facts {
if capped.contains(&fact.subject) || capped.contains(&fact.object) {
continue;
}
batch.insert_edge(&fact.predicate, &fact.subject, &fact.object);
}
batch.commit()?;
let derived_after = db
.node_edges(&key)?
.into_iter()
.filter(|e| e.derived)
.count();
report.derived = derived_after.saturating_sub(derived_before);
let mut touched = entity_key_list;
touched.extend(report.provisional.iter().cloned());
let same_as_after = same_as_pairs(db, &touched);
report.same_as_lost = same_as_lost(&same_as_before, &same_as_after);
report.same_as = same_as_after
.into_iter()
.filter(|p| !same_as_before.iter().any(|b| b.a == p.a && b.b == p.b))
.collect();
Ok(report)
}
fn stub_props(key: &str) -> Vec<(String, Value)> {
let mut aliases = derive_aliases(key, Some(key));
if aliases.len() > MAX_ALIASES {
aliases = derive_aliases(key, None);
}
vec![
(NAME_FIELD.to_string(), Value::Str(key.to_string())),
(PROVISIONAL_PROP.to_string(), Value::Bool(true)),
(ALIASES_FIELD.to_string(), aliases_value(&aliases)),
]
}
fn note_key(ts: i64, text: &str) -> String {
const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
let mut h = FNV_OFFSET;
for b in ts.to_string().bytes().chain(text.bytes()) {
h ^= u64::from(b);
h = h.wrapping_mul(FNV_PRIME);
}
format!("note:{h:016x}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_key_is_a_function_of_ts_and_text_alone() {
assert_eq!(note_key(1, "a"), note_key(1, "a"));
assert_ne!(note_key(1, "a"), note_key(2, "a"));
assert_ne!(note_key(1, "a"), note_key(1, "b"));
assert!(note_key(1, "a").strip_prefix("note:").unwrap().len() == 16);
}
}