#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExtractedFact {
pub text: String,
pub entities: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExtractedRelation {
pub subject: String,
pub predicate: String,
pub object: String,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct ExtractedAttribute {
pub entity: String,
pub key: String,
pub value: serde_json::Value,
}
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct Extraction {
pub facts: Vec<ExtractedFact>,
pub relations: Vec<ExtractedRelation>,
pub attributes: Vec<ExtractedAttribute>,
}
const KINSHIP_NOUNS: &[(&str, &str)] = &[
("pere", "pere"),
("mere", "mere"),
("frere", "frere"),
("soeur", "soeur"),
("fils", "fils"),
("fille", "fille"),
("oncle", "oncle"),
("tante", "tante"),
("cousin", "cousin"),
("cousine", "cousine"),
("neveu", "neveu"),
("niece", "niece"),
("grand-pere", "grand-pere"),
("grand-mere", "grand-mere"),
("grand-oncle", "grand-oncle"),
("grand-tante", "grand-tante"),
("arriere-grand-pere", "arriere-grand-pere"),
("arriere-grand-mere", "arriere-grand-mere"),
("petit-fils", "petit-fils"),
("petite-fille", "petite-fille"),
("beau-pere", "beau-pere"),
("belle-mere", "belle-mere"),
("beau-frere", "beau-frere"),
("belle-soeur", "belle-soeur"),
("beau-fils", "beau-fils"),
("belle-fille", "belle-fille"),
("gendre", "beau-fils"),
("bru", "belle-fille"),
("demi-frere", "demi-frere"),
("demi-soeur", "demi-soeur"),
("parrain", "parrain"),
("marraine", "marraine"),
("filleul", "filleul"),
("filleule", "filleule"),
("epoux", "epoux"),
("epouse", "epouse"),
("mari", "epoux"),
("femme", "epouse"),
("father", "pere"),
("mother", "mere"),
("brother", "frere"),
("sister", "soeur"),
("son", "fils"),
("daughter", "fille"),
("uncle", "oncle"),
("aunt", "tante"),
("nephew", "neveu"),
("grandfather", "grand-pere"),
("grandmother", "grand-mere"),
("grandson", "petit-fils"),
("granddaughter", "petite-fille"),
("husband", "epoux"),
("wife", "epouse"),
("father-in-law", "beau-pere"),
("mother-in-law", "belle-mere"),
("brother-in-law", "beau-frere"),
("sister-in-law", "belle-soeur"),
("son-in-law", "beau-fils"),
("daughter-in-law", "belle-fille"),
("stepfather", "beau-pere"),
("stepmother", "belle-mere"),
("stepbrother", "beau-frere"),
("stepsister", "belle-soeur"),
("half-brother", "demi-frere"),
("half-sister", "demi-soeur"),
("godfather", "parrain"),
("godmother", "marraine"),
("godson", "filleul"),
("goddaughter", "filleule"),
];
const POSSESSIVE_MARKERS: &[&str] = &[
" a un ",
" a une ",
" a pour ",
" a des ",
" a deux ",
" a trois ",
" a quatre ",
" has a ",
" has an ",
" has two ",
" has three ",
" has four ",
];
const GENITIVE_LINKS: &[&str] = &[" de ", " d'", " of "];
const SAXON_MARKER: &str = "'s ";
const GENITIVE_COPULAS: &[&str] = &[" est ", " sont ", " is ", " are "];
const LEADING_ARTICLES: &[&str] = &["le ", "la ", "les ", "l'", "the "];
const ENUMERATION_SEPARATORS: &[&str] = &[", et ", ", and ", " et ", " and ", " & ", ", "];
const FOLDINGS: &[(char, &str)] = &[
('à', "a"),
('â', "a"),
('ä', "a"),
('é', "e"),
('è', "e"),
('ê', "e"),
('ë', "e"),
('î', "i"),
('ï', "i"),
('ô', "o"),
('ö', "o"),
('ù', "u"),
('û', "u"),
('ü', "u"),
('ç', "c"),
('œ', "oe"),
('æ', "ae"),
('\u{2019}', "'"),
];
fn fold(text: &str) -> String {
let mut folded = String::with_capacity(text.len());
for ch in text.chars().flat_map(char::to_lowercase) {
match FOLDINGS.iter().find(|(from, _)| *from == ch) {
Some((_, to)) => folded.push_str(to),
None => folded.push(ch),
}
}
folded
}
struct Kinship {
noun: &'static str,
holder: String,
bearers: Vec<String>,
}
fn word_prefix_len(rest: &str, word: &str) -> Option<usize> {
let tail = rest.strip_prefix(word)?;
let (tail, plural) = match tail.strip_prefix('s') {
Some(shorter) => (shorter, 1),
None => (tail, 0),
};
let glued = |ch: char| ch.is_alphanumeric() || ch == '-';
(!tail.starts_with(glued)).then_some(word.len() + plural)
}
fn ends_with_word(head: &str, word: &str) -> bool {
ends_exactly(head, word)
|| head
.strip_suffix('s')
.is_some_and(|singular| ends_exactly(singular, word))
}
fn ends_exactly(head: &str, word: &str) -> bool {
head.strip_suffix(word)
.is_some_and(|lead| !lead.ends_with(|ch: char| ch.is_alphanumeric() || ch == '-'))
}
fn noun_at(text: &str) -> Option<(&'static str, usize)> {
KINSHIP_NOUNS.iter().find_map(|(spelling, canonical)| {
word_prefix_len(text, spelling).map(|len| (*canonical, len))
})
}
fn noun_before(head: &str) -> Option<&'static str> {
KINSHIP_NOUNS
.iter()
.find(|(spelling, _)| ends_with_word(head, spelling))
.map(|(_, canonical)| *canonical)
}
fn strip_any<'a>(text: &'a str, prefixes: &[&str]) -> Option<&'a str> {
prefixes.iter().find_map(|prefix| text.strip_prefix(prefix))
}
fn endpoint_names(relations: &[ExtractedRelation]) -> Vec<String> {
let mut names: Vec<String> = relations
.iter()
.flat_map(|relation| [relation.subject.clone(), relation.object.clone()])
.collect();
names.sort_unstable();
names.dedup();
names
}
fn holder_of(before: &str, names: &[String]) -> Option<String> {
names
.iter()
.filter_map(|name| before.rfind(&fold(name)).map(|at| (at, name)))
.max_by_key(|(at, _)| *at)
.map(|(_, name)| name.clone())
}
fn name_at(text: &str, names: &[String]) -> Option<String> {
names
.iter()
.filter(|name| text.starts_with(&fold(name)))
.max_by_key(|name| name.len())
.cloned()
}
fn name_before(head: &str, names: &[String]) -> Option<String> {
names
.iter()
.filter(|name| head.ends_with(&fold(name)))
.max_by_key(|name| name.len())
.cloned()
}
fn first_name(text: &str, names: &[String]) -> Option<(usize, String)> {
names
.iter()
.filter_map(|name| text.find(&fold(name)).map(|at| (at, name)))
.min_by_key(|(at, name)| (*at, std::cmp::Reverse(name.len())))
.map(|(at, name)| (at, name.clone()))
}
fn enumeration_from(text: &str, first: String, names: &[String]) -> Vec<String> {
let mut rest = &text[fold(&first).len()..];
let mut bearers = vec![first];
while let Some((name, tail)) = next_enumerated(rest, names) {
bearers.push(name);
rest = tail;
}
bearers
}
const CLAUSE_VERBS: &[&str] = &[
" est ",
" sont ",
" etait ",
" etaient ",
" a ",
" ont ",
" avait ",
" avaient ",
" is ",
" are ",
" was ",
" were ",
" has ",
" have ",
" had ",
];
fn next_enumerated<'a>(rest: &'a str, names: &[String]) -> Option<(String, &'a str)> {
let tail = strip_any(rest, ENUMERATION_SEPARATORS)?;
let name = name_at(tail, names)?;
let cut = fold(&name).len();
let after = &tail[cut..];
if CLAUSE_VERBS.iter().any(|verb| after.starts_with(verb)) {
return None;
}
Some((name, after))
}
fn bearers_after(after: &str, names: &[String]) -> Vec<String> {
match first_name(after, names) {
Some((at, first)) => enumeration_from(&after[at..], first, names),
None => Vec::new(),
}
}
fn bearers_at(text: &str, names: &[String]) -> Vec<String> {
[Some(text), strip_any(text, LEADING_ARTICLES)]
.into_iter()
.flatten()
.find_map(|text| name_at(text, names).map(|first| enumeration_from(text, first, names)))
.unwrap_or_default()
}
fn find_possessive(folded: &str, names: &[String]) -> Option<Kinship> {
let (start, noun, end) = POSSESSIVE_MARKERS
.iter()
.filter_map(|marker| folded.find(marker).map(|at| at + marker.len()))
.filter_map(|start| {
let (noun, len) = noun_at(folded.get(start..)?)?;
Some((start, noun, start + len))
})
.min_by_key(|(start, _, _)| *start)?;
Some(Kinship {
noun,
holder: holder_of(folded.get(..start)?, names)?,
bearers: bearers_after(folded.get(end..)?, names),
})
}
fn find_genitive(folded: &str, names: &[String]) -> Option<Kinship> {
find_of_genitive(folded, names).or_else(|| find_saxon_genitive(folded, names))
}
fn find_of_genitive(folded: &str, names: &[String]) -> Option<Kinship> {
let mut links: Vec<(usize, usize)> = GENITIVE_LINKS
.iter()
.flat_map(|link| folded.match_indices(link).map(|(at, m)| (at, m.len())))
.collect();
links.sort_unstable();
links
.into_iter()
.find_map(|(at, len)| of_genitive_at(folded, at, len, names))
}
fn of_genitive_at(folded: &str, at: usize, len: usize, names: &[String]) -> Option<Kinship> {
let noun = noun_before(folded.get(..at)?)?;
let after_link = folded.get(at + len..)?;
let holder = name_at(after_link, names)?;
let after_holder = after_link.get(fold(&holder).len()..)?;
let bearers = bearers_at(strip_any(after_holder, GENITIVE_COPULAS)?, names);
Some(Kinship {
noun,
holder,
bearers,
})
}
fn find_saxon_genitive(folded: &str, names: &[String]) -> Option<Kinship> {
folded
.match_indices(SAXON_MARKER)
.find_map(|(at, marker)| saxon_genitive_at(folded, at, marker.len(), names))
}
fn saxon_genitive_at(folded: &str, at: usize, len: usize, names: &[String]) -> Option<Kinship> {
let holder = name_before(folded.get(..at)?, names)?;
let (noun, noun_len) = noun_at(folded.get(at + len..)?)?;
let after_noun = folded.get(at + len + noun_len..)?;
let bearers = bearers_at(strip_any(after_noun, GENITIVE_COPULAS)?, names);
Some(Kinship {
noun,
holder,
bearers,
})
}
fn find_kinship(folded: &str, names: &[String]) -> Option<Kinship> {
find_possessive(folded, names).or_else(|| find_genitive(folded, names))
}
fn predicate_stem(predicate: &str) -> String {
fold(predicate)
.split_whitespace()
.next()
.unwrap_or_default()
.to_string()
}
fn predicate_noun(predicate: &str) -> Option<&'static str> {
let stem = predicate_stem(predicate);
KINSHIP_NOUNS
.iter()
.find(|(spelling, _)| word_prefix_len(&stem, spelling) == Some(stem.len()))
.map(|(_, canonical)| *canonical)
}
fn joins(relation: &ExtractedRelation, one: &str, other: &str) -> bool {
(relation.subject == one && relation.object == other)
|| (relation.subject == other && relation.object == one)
}
fn reorient(relation: &mut ExtractedRelation, noun: &str, holder: &str, bearer: &str) {
let Some(stem) = predicate_noun(&relation.predicate) else {
return;
};
if !joins(relation, holder, bearer) {
return;
}
let (subject, object) = if stem == noun {
(bearer, holder)
} else {
(holder, bearer)
};
relation.subject = subject.to_string();
relation.object = object.to_string();
}
pub(crate) fn orient_kinship(passage: &str, relations: &mut [ExtractedRelation]) {
let folded = fold(passage);
let names = endpoint_names(relations);
let Some(kinship) = find_kinship(&folded, &names) else {
return;
};
for bearer in &kinship.bearers {
if *bearer == kinship.holder {
continue;
}
for relation in relations.iter_mut() {
reorient(relation, kinship.noun, &kinship.holder, bearer);
}
}
}
#[derive(Debug, thiserror::Error)]
#[non_exhaustive] pub enum ExtractError {
#[error("extraction backend error: {0}")]
Backend(String),
#[error("could not parse facts from extractor output: {0}")]
Parse(String),
}
pub trait Extractor {
fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>, ExtractError>;
fn extract_graph(&self, text: &str) -> Result<Extraction, ExtractError> {
Ok(Extraction {
facts: self.extract(text)?,
..Extraction::default()
})
}
}
impl<T: Extractor + ?Sized> Extractor for std::sync::Arc<T> {
fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>, ExtractError> {
(**self).extract(text)
}
fn extract_graph(&self, text: &str) -> Result<Extraction, ExtractError> {
(**self).extract_graph(text)
}
}
pub type DynExtractor = std::sync::Arc<dyn Extractor + Send + Sync>;
#[derive(Debug, Clone, Copy, Default)]
pub struct OutlineExtractor;
fn directive_fields(rest: &str) -> Vec<&str> {
rest.split('|').map(str::trim).collect()
}
fn wrong_field_count(kind: &str, expected: usize, given: usize) -> ExtractError {
ExtractError::Parse(format!(
"`{kind}:` takes {expected} `|`-separated fields, {given} given"
))
}
fn parse_edge(rest: &str) -> Result<ExtractedRelation, ExtractError> {
let fields = directive_fields(rest);
let [subject, predicate, object] = fields[..] else {
return Err(wrong_field_count("edge", 3, fields.len()));
};
if subject.is_empty() || predicate.is_empty() || object.is_empty() {
return Err(ExtractError::Parse(
"`edge:` takes a non-blank subject, predicate and object".to_owned(),
));
}
Ok(ExtractedRelation {
subject: crate::service::canonical_entity_name(subject),
predicate: predicate.to_owned(),
object: crate::service::canonical_entity_name(object),
})
}
fn parse_attr(rest: &str) -> Result<ExtractedAttribute, ExtractError> {
let fields = directive_fields(rest);
let [entity, key, value] = fields[..] else {
return Err(wrong_field_count("attr", 3, fields.len()));
};
if entity.is_empty() || key.is_empty() {
return Err(ExtractError::Parse(
"`attr:` takes a non-blank entity and key".to_owned(),
));
}
let value = serde_json::from_str(value)
.map_err(|err| ExtractError::Parse(format!("`attr:` value is not JSON: {err}")))?;
Ok(ExtractedAttribute {
entity: crate::service::canonical_entity_name(entity),
key: key.to_owned(),
value,
})
}
fn parse_fact(body: &str) -> Result<ExtractedFact, ExtractError> {
let (text, topics) = body.split_once('|').unwrap_or((body, ""));
let text = text.trim();
if text.is_empty() {
return Err(ExtractError::Parse(
"a fact line takes a non-blank text".to_owned(),
));
}
Ok(ExtractedFact {
text: text.to_owned(),
entities: topics
.split(',')
.map(crate::service::canonical_entity_name)
.filter(|topic| !topic.is_empty())
.collect(),
})
}
impl Extractor for OutlineExtractor {
fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>, ExtractError> {
Ok(self.extract_graph(text)?.facts)
}
fn extract_graph(&self, text: &str) -> Result<Extraction, ExtractError> {
let mut extraction = Extraction::default();
for line in text.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
if let Some(rest) = line.strip_prefix("edge:") {
extraction.relations.push(parse_edge(rest)?);
} else if let Some(rest) = line.strip_prefix("attr:") {
extraction.attributes.push(parse_attr(rest)?);
} else {
extraction
.facts
.push(parse_fact(line.strip_prefix("fact:").unwrap_or(line))?);
}
}
Ok(extraction)
}
}
pub enum ExtractorSelection {
Disabled,
Ready(DynExtractor),
NeedsRemoteConfig(&'static str),
}
impl std::fmt::Debug for ExtractorSelection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Disabled => f.write_str("Disabled"),
Self::Ready(_) => f.write_str("Ready(<extractor>)"),
Self::NeedsRemoteConfig(name) => write!(f, "NeedsRemoteConfig({name})"),
}
}
}
pub fn select_extractor(backend: &str) -> Result<ExtractorSelection, String> {
match backend {
"outline" => Ok(ExtractorSelection::Ready(std::sync::Arc::new(
OutlineExtractor,
))),
"ollama" => Ok(ExtractorSelection::NeedsRemoteConfig("ollama")),
"openai" => Ok(ExtractorSelection::NeedsRemoteConfig("openai")),
"none" | "" => Ok(ExtractorSelection::Disabled),
other => Err(format!(
"unknown extraction backend '{other}' (expected 'outline' for the \
offline deterministic reader, 'ollama' for a local generative \
model, 'openai' for any OpenAI-compatible server — oMLX, \
llama.cpp, LM Studio, vLLM or a hosted provider, selected by URL \
rather than by name — or 'none')"
)),
}
}
#[cfg(feature = "extractor-http")]
pub const DEFAULT_OLLAMA_URL: &str = "http://localhost:11434";
#[cfg(feature = "extractor-http")]
const REQUEST_TIMEOUT_SECS: u64 = 300;
#[cfg(feature = "extractor-http")]
const MAX_GENERATION_TOKENS: u32 = 512;
#[cfg(feature = "extractor-http")]
const EXTRACT_LEVERS: crate::http_retry::FailureLevers<'static> =
crate::http_retry::FailureLevers {
url_var: "VELESDB_MEMORY_EXTRACTOR_URL",
model_var: "VELESDB_MEMORY_EXTRACTOR_MODEL",
fallback: None,
};
#[cfg(feature = "extractor-http")]
enum GenerateCall {
Transport(Box<ureq::Error>),
Body(std::io::Error),
}
#[cfg(feature = "extractor-http")]
fn generate_is_retryable(err: &GenerateCall) -> bool {
match err {
GenerateCall::Transport(inner) => crate::http_retry::is_retryable(inner),
GenerateCall::Body(inner) => crate::http_retry::io_is_retryable(inner),
}
}
#[cfg(feature = "extractor-http")]
fn describe_generate_failure(url: &str, model: &str, err: &GenerateCall, attempts: u32) -> String {
let cause = match err {
GenerateCall::Transport(inner) => inner.to_string(),
GenerateCall::Body(inner) => format!("reading the response failed: {inner}"),
};
crate::http_retry::actionable_ollama_failure(
"generate",
url,
model,
attempts,
&cause,
&EXTRACT_LEVERS,
)
}
#[cfg(feature = "extractor-http")]
#[derive(Debug, Clone)]
pub struct OllamaExtractor {
base_url: String,
model: String,
agent: ureq::Agent,
}
#[cfg(feature = "extractor-http")]
impl OllamaExtractor {
#[must_use]
pub fn new(base_url: impl Into<String>, model: impl Into<String>) -> Self {
let agent =
crate::http_client::bounded_agent(crate::http_client::AgentBudget::local_daemon(
std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS),
));
Self {
base_url: base_url.into(),
model: model.into(),
agent,
}
}
}
#[cfg(feature = "extractor-http")]
fn facts_from_reply(reply: &str) -> Result<Vec<ExtractedFact>, ExtractError> {
let raw =
json_slice::<Vec<RawFact>>(reply).ok_or_else(|| ExtractError::Parse(truncate(reply)))?;
Ok(raw.into_iter().filter_map(RawFact::into_fact).collect())
}
#[cfg(feature = "extractor-http")]
fn extraction_from_reply(reply: &str) -> Result<Extraction, ExtractError> {
let raw = json_slice_object::<RawExtraction>(reply)
.ok_or_else(|| ExtractError::Parse(truncate(reply)))?;
Ok(raw.into_extraction())
}
#[cfg(feature = "extractor-http")]
impl Extractor for OllamaExtractor {
fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>, ExtractError> {
facts_from_reply(&self.generate(&build_prompt(text))?)
}
fn extract_graph(&self, text: &str) -> Result<Extraction, ExtractError> {
extraction_from_reply(&self.generate(&build_graph_prompt(text))?)
}
}
#[cfg(feature = "extractor-http")]
#[derive(Debug)]
pub struct OpenAiExtractor {
client: crate::http_client::HttpJsonClient,
model: String,
}
#[cfg(feature = "extractor-http")]
impl OpenAiExtractor {
#[must_use]
pub fn new(
base_url: impl Into<String>,
model: impl Into<String>,
auth: crate::http_client::Auth,
) -> Self {
let agent =
crate::http_client::bounded_agent(crate::http_client::AgentBudget::local_daemon(
std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS),
));
Self {
client: crate::http_client::HttpJsonClient::new(
crate::openai::base_url(&base_url.into()),
auth,
agent,
),
model: model.into(),
}
}
fn generate(&self, prompt: &str) -> Result<String, ExtractError> {
let body = crate::openai::chat_body(&self.model, prompt, MAX_GENERATION_TOKENS);
let payload = self
.client
.post_json(crate::openai::CHAT_COMPLETIONS_PATH, &body)
.map_err(|failure| {
ExtractError::Backend(crate::http_retry::actionable_openai_failure(
"chat/completions",
&failure.url,
&self.model,
failure.attempts,
&failure.cause,
Some(
"use the offline deterministic reader with \
VELESDB_MEMORY_EXTRACTOR=outline",
),
))
})?;
crate::openai::parse_chat_response(&payload).map_err(ExtractError::Backend)
}
}
#[cfg(feature = "extractor-http")]
impl Extractor for OpenAiExtractor {
fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>, ExtractError> {
facts_from_reply(&self.generate(&build_prompt(text))?)
}
fn extract_graph(&self, text: &str) -> Result<Extraction, ExtractError> {
extraction_from_reply(&self.generate(&build_graph_prompt(text))?)
}
}
#[cfg(feature = "extractor-http")]
impl OllamaExtractor {
fn generate(&self, prompt: &str) -> Result<String, ExtractError> {
let url = format!("{}/api/generate", self.base_url);
let body = serde_json::json!({
"model": self.model,
"prompt": prompt,
"stream": false,
"think": false,
"keep_alive": crate::embedder::keep_alive(),
"options": { "temperature": 0, "num_predict": MAX_GENERATION_TOKENS },
})
.to_string();
let attempt = || {
let response = self
.agent
.post(&url)
.set("Content-Type", "application/json")
.send_string(&body)
.map_err(|err| GenerateCall::Transport(Box::new(err)))?;
response.into_string().map_err(GenerateCall::Body)
};
let payload = crate::http_retry::with_retry(
&crate::http_retry::HTTP_RETRIES,
generate_is_retryable,
attempt,
)
.map_err(|(err, attempts)| {
ExtractError::Backend(describe_generate_failure(&url, &self.model, &err, attempts))
})?;
parse_generate_response(&payload)
}
}
#[cfg(feature = "extractor-http")]
#[derive(serde::Deserialize)]
struct RawFact {
fact: String,
#[serde(default)]
entities: Vec<String>,
}
#[cfg(feature = "extractor-http")]
impl RawFact {
fn into_fact(self) -> Option<ExtractedFact> {
let text = self.fact.trim().to_string();
if text.is_empty() {
return None;
}
let mut entities: Vec<String> = self
.entities
.into_iter()
.map(|entity| entity.trim().to_lowercase())
.filter(|entity| !entity.is_empty())
.collect();
entities.sort_unstable();
entities.dedup();
Some(ExtractedFact { text, entities })
}
}
#[cfg(feature = "extractor-http")]
fn canonical_entity(name: &str) -> String {
name.trim().to_lowercase()
}
#[cfg(feature = "extractor-http")]
#[derive(serde::Deserialize)]
struct RawExtraction {
#[serde(default)]
facts: Vec<RawFact>,
#[serde(default)]
relations: Vec<RawRelation>,
#[serde(default)]
attributes: Vec<RawAttribute>,
}
#[cfg(feature = "extractor-http")]
#[derive(serde::Deserialize)]
struct RawRelation {
subject: String,
predicate: String,
object: String,
}
#[cfg(feature = "extractor-http")]
#[derive(serde::Deserialize)]
struct RawAttribute {
entity: String,
key: String,
value: serde_json::Value,
}
#[cfg(feature = "extractor-http")]
impl RawExtraction {
fn into_extraction(self) -> Extraction {
Extraction {
facts: self
.facts
.into_iter()
.filter_map(RawFact::into_fact)
.collect(),
relations: self
.relations
.into_iter()
.filter_map(RawRelation::into_relation)
.collect(),
attributes: self
.attributes
.into_iter()
.filter_map(RawAttribute::into_attribute)
.collect(),
}
}
}
#[cfg(feature = "extractor-http")]
impl RawRelation {
fn into_relation(self) -> Option<ExtractedRelation> {
let subject = canonical_entity(&self.subject);
let object = canonical_entity(&self.object);
let predicate = self.predicate.trim().to_string();
if subject.is_empty() || object.is_empty() || predicate.is_empty() || subject == object {
return None;
}
Some(ExtractedRelation {
subject,
predicate,
object,
})
}
}
#[cfg(feature = "extractor-http")]
impl RawAttribute {
fn into_attribute(self) -> Option<ExtractedAttribute> {
let entity = canonical_entity(&self.entity);
let key = self.key.trim().to_string();
if entity.is_empty() || key.is_empty() || self.value.is_null() {
return None;
}
Some(ExtractedAttribute {
entity,
key,
value: self.value,
})
}
}
#[cfg(feature = "extractor-http")]
fn build_graph_prompt(text: &str) -> String {
format!(
"You are building a knowledge graph from the passage below.\n\n\
Passage:\n{text}\n\n\
STEP 0 — Identify the passage's language. Everything you write (facts, \
predicates, attribute keys) MUST be in THAT language. Do not copy the language \
of the examples below: they are shown in several languages on purpose, and you \
must match the PASSAGE, never the example.\n\n\
Return THREE things.\n\n\
1. \"facts\": the atomic, standalone facts a person would remember, in the \
passage's language. Rewrite each as a self-contained sentence (resolve \
pronouns to names; keep absolute dates). For each, list 1-4 key TOPICS it \
concerns, as short canonical lowercase noun phrases, so the same topic recurs \
as the SAME tag across passages.\n\n\
2. \"relations\": every explicit relationship BETWEEN TWO NAMED ENTITIES, as \
subject/predicate/object triples. Use the entity's full name, lowercase \
(e.g. \"bruno durand\").\n\
The predicate is a LABEL, not a sentence: **at most 3 words**, lowercase, in \
the passage's language. Examples of the SHAPE, each in its own language — \
match the passage, not these: a French passage gives \"travaille chez\", \
\"pere de\"; an English passage gives \"works at\", \"father of\"; a Spanish \
passage gives \"trabaja en\". NEVER restate the sentence — write \"surveille \
les fuites\", not \"est utilise pour la surveillance de fuites de donnees\". \
If you cannot say it in 3 words, pick the closest short label.\n\
DIRECTION: the subject is whoever CARRIES the relation, not the subject of the \
sentence. \"A a une soeur, B\" means B is A's sister, so the triple is \
B/\"soeur de\"/A — never A/\"soeur de\"/B. \"A has a brother, B\" means B is \
A's brother: B/\"brother of\"/A. Same for every possessive.\n\
Never emit both directions of the SAME predicate over the same pair — \
\"X brother of Y\" plus \"Y brother of X\" is a contradiction, not a \
converse: emit exactly one. But two DIFFERENT predicates the passage states \
separately over the same pair (\"A possede B\" then \"B appartient a A\") \
are two stated facts — keep both.\n\
Every named entity the passage RELATES to another must appear in at least one \
triple — an entity that only receives attributes and no edge is a dead end in \
the graph.\n\n\
3. \"attributes\": every property a named entity HAS, as entity/key/value. Use \
short lowercase keys in the passage's language (\"age\", \"ville\", \
\"employeur\"). Emit numbers as JSON NUMBERS, never strings: 15, not \"15\". \
Omit anything the passage does not state.\n\n\
Return ONLY this JSON object, no prose, no markdown fence:\n\
{{\"facts\": [{{\"fact\": string, \"entities\": [string]}}], \
\"relations\": [{{\"subject\": string, \"predicate\": string, \"object\": string}}], \
\"attributes\": [{{\"entity\": string, \"key\": string, \"value\": string|number|boolean}}]}}"
)
}
#[cfg(feature = "extractor-http")]
fn build_prompt(text: &str) -> String {
format!(
"You are building a memory graph from the passage below.\n\n\
Passage:\n{text}\n\n\
Extract the atomic, standalone facts a person would remember. Rewrite each as a \
self-contained sentence (resolve pronouns to names; keep absolute dates). For \
each fact also list 1-4 key TOPICS it concerns: the recurring subjects, \
activities, events, interests, plans, places, organisations, or named people a \
later question might reference. Use short, canonical, lowercase noun phrases \
(e.g. \"adoption\", \"charity race\", \"therapy\", \"new job\") so the same topic \
recurs as the SAME tag across passages.\n\n\
Return ONLY a JSON array, no prose, each item exactly:\n\
{{\"fact\": string, \"entities\": [string]}}"
)
}
#[cfg(feature = "extractor-http")]
fn parse_generate_response(body: &str) -> Result<String, ExtractError> {
let value: serde_json::Value = serde_json::from_str(body)
.map_err(|err| ExtractError::Backend(format!("invalid generate response: {err}")))?;
let text = value
.get("response")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| ExtractError::Backend("ollama reply had no `response` field".to_string()))?;
Ok(text.trim().to_string())
}
#[cfg(feature = "extractor-http")]
fn truncate(text: &str) -> String {
const LIMIT: usize = 120;
let mut out = String::new();
for word in text.split_whitespace() {
let sep_len = usize::from(!out.is_empty());
if out.len() + sep_len + word.len() > LIMIT {
break;
}
if !out.is_empty() {
out.push(' ');
}
out.push_str(word);
}
out
}
#[cfg(feature = "extractor-http")]
fn json_slice<T: serde::de::DeserializeOwned>(text: &str) -> Option<T> {
let slice = balanced_slice(text)?;
serde_json::from_str::<T>(slice).ok()
}
#[cfg(feature = "extractor-http")]
fn json_slice_object<T: serde::de::DeserializeOwned>(text: &str) -> Option<T> {
let slice = balanced_slice_preferring(text, b'{')?;
serde_json::from_str::<T>(slice).ok()
}
#[cfg(feature = "extractor-http")]
fn balanced_slice(text: &str) -> Option<&str> {
balanced_slice_preferring(text, b'[')
}
#[cfg(feature = "extractor-http")]
fn balanced_slice_preferring(text: &str, preferred: u8) -> Option<&str> {
let bytes = text.as_bytes();
let fallback = if preferred == b'[' { b'{' } else { b'[' };
let start = bytes
.iter()
.position(|&b| b == preferred)
.or_else(|| bytes.iter().position(|&b| b == fallback))?;
let open = bytes[start];
let close = if open == b'[' { b']' } else { b'}' };
let mut depth = 0u32;
let mut in_string = false;
let mut escaped = false;
for (offset, &byte) in bytes[start..].iter().enumerate() {
if in_string {
in_string = step_string(&mut escaped, byte);
} else if scan_structural(byte, open, close, &mut in_string, &mut depth) {
return Some(&text[start..=start + offset]);
}
}
None
}
#[cfg(feature = "extractor-http")]
fn scan_structural(byte: u8, open: u8, close: u8, in_string: &mut bool, depth: &mut u32) -> bool {
if byte == b'"' {
*in_string = true;
} else if byte == open {
*depth += 1;
} else if byte == close {
*depth = depth.saturating_sub(1);
return *depth == 0;
}
false
}
#[cfg(feature = "extractor-http")]
fn step_string(escaped: &mut bool, byte: u8) -> bool {
match (*escaped, byte) {
(true, _) => {
*escaped = false;
true
}
(false, b'\\') => {
*escaped = true;
true
}
(false, b'"') => false,
(false, _) => true,
}
}
#[cfg(test)]
#[path = "extractor_selection_tests.rs"]
mod selection_tests;
#[cfg(all(test, feature = "extractor-http"))]
#[path = "extract_tests.rs"]
mod tests;