use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use crate::core::bm25_index::{BM25Index, SearchResult};
use crate::core::context_field::{ContextItemId, ContextState, Provenance, ViewCosts};
use crate::core::context_ledger::{ContextLedger, LedgerEntry};
use crate::core::knowledge::ProjectKnowledge;
use crate::core::session::SessionState;
use crate::core::tokens::count_tokens;
use super::types::{
CandidateProvider, ContextObjectKind, ContextObjectV1, Freshness, RetrievalContext,
SensitivityLevel, SideEffectPolicy,
};
const KNOWLEDGE_PROVIDER: &str = "knowledge.facts";
const SESSION_PROVIDER: &str = "session.state";
const LEDGER_PROVIDER: &str = "context.ledger";
const SEARCH_PROVIDER: &str = "index.bm25";
const GRAPH_PROVIDER: &str = "index.graph";
const GRAPH_NEIGHBOURS_PER_SEED: usize = 3;
pub struct KnowledgeProvider {
project_root: String,
}
impl KnowledgeProvider {
pub fn new(project_root: impl Into<String>) -> Self {
Self {
project_root: project_root.into(),
}
}
}
impl CandidateProvider for KnowledgeProvider {
fn provider_id(&self) -> &str {
KNOWLEDGE_PROVIDER
}
fn candidates(&self, ctx: &RetrievalContext) -> Vec<ContextObjectV1> {
let Some(mut knowledge) = ProjectKnowledge::load(&self.project_root) else {
return Vec::new();
};
let (facts, _) = knowledge.recall_for_output(&ctx.query, ctx.max_candidates);
let deliverable = knowledge.delivery_filter();
facts
.into_iter()
.filter(|fact| deliverable(fact))
.map(|fact| {
let mut metadata = HashMap::new();
metadata.insert("category".to_string(), fact.category.clone());
metadata.insert("key".to_string(), fact.key.clone());
metadata.insert("source_session".to_string(), fact.source_session.clone());
context_object(
ContextItemId::from_knowledge(&fact.category, &fact.key),
ContextObjectKind::Fact,
KNOWLEDGE_PROVIDER,
format!("knowledge:{}:{}", fact.category, fact.key),
format!("{}: {}", fact.category, fact.key),
Some(fact.value.clone()),
freshness(fact.created_at.to_rfc3339(), false),
fact.confidence,
count_tokens(&fact.value),
ViewCosts::from_full_tokens(count_tokens(&fact.value)),
Provenance::default(),
metadata,
)
})
.collect()
}
fn side_effect_policy(&self) -> SideEffectPolicy {
SideEffectPolicy::MutatesStats
}
}
pub(crate) struct SessionProvider {
project_root: String,
caller: Option<crate::core::session::SharedSession>,
}
impl SessionProvider {
pub(crate) fn new(project_root: impl Into<String>) -> Self {
Self {
project_root: project_root.into(),
caller: crate::core::session::caller::current(),
}
}
}
impl CandidateProvider for SessionProvider {
fn provider_id(&self) -> &str {
SESSION_PROVIDER
}
fn candidates(&self, ctx: &RetrievalContext) -> Vec<ContextObjectV1> {
let caller = self.caller.clone();
crate::core::session::caller::scope(caller, || {
crate::core::session::caller::with_for_project(&self.project_root, |session| {
Self::of_session(session, ctx.max_candidates)
})
})
.unwrap_or_default()
}
fn side_effect_policy(&self) -> SideEffectPolicy {
SideEffectPolicy::ReadOnly
}
}
impl SessionProvider {
fn of_session(session: &SessionState, max_candidates: usize) -> Vec<ContextObjectV1> {
let mut candidates = Vec::new();
for finding in session
.findings
.iter()
.filter(|finding| session.finding_served(finding))
{
let mut metadata = HashMap::new();
if let Some(file) = &finding.file {
metadata.insert("file".to_string(), file.clone());
}
if let Some(line) = finding.line {
metadata.insert("line".to_string(), line.to_string());
}
let id = ContextItemId::from_provider(
SESSION_PROVIDER,
&format!(
"finding:{}",
finding.timestamp.timestamp_nanos_opt().unwrap_or_default()
),
);
candidates.push((
finding.timestamp,
context_object(
id,
ContextObjectKind::SessionItem,
SESSION_PROVIDER,
format!("session:{}:finding", session.id),
"Session finding".to_string(),
Some(finding.summary.clone()),
freshness(finding.timestamp.to_rfc3339(), false),
1.0,
count_tokens(&finding.summary),
ViewCosts::from_full_tokens(count_tokens(&finding.summary)),
Provenance::default(),
metadata,
),
));
}
for decision in &session.decisions {
let mut metadata = HashMap::new();
if let Some(rationale) = &decision.rationale {
metadata.insert("rationale".to_string(), rationale.clone());
}
let id = ContextItemId::from_provider(
SESSION_PROVIDER,
&format!(
"decision:{}",
decision.timestamp.timestamp_nanos_opt().unwrap_or_default()
),
);
candidates.push((
decision.timestamp,
context_object(
id,
ContextObjectKind::SessionItem,
SESSION_PROVIDER,
format!("session:{}:decision", session.id),
"Session decision".to_string(),
Some(decision.summary.clone()),
freshness(decision.timestamp.to_rfc3339(), false),
1.0,
count_tokens(&decision.summary),
ViewCosts::from_full_tokens(count_tokens(&decision.summary)),
Provenance::default(),
metadata,
),
));
}
for file in session.files_touched.iter().filter(|file| file.modified) {
let summary = file
.summary
.clone()
.unwrap_or_else(|| format!("Modified file: {}", file.path));
let mut metadata = HashMap::new();
metadata.insert("path".to_string(), file.path.clone());
metadata.insert("mode".to_string(), file.last_mode.clone());
candidates.push((
session.updated_at,
context_object(
ContextItemId::from_file(&file.path),
ContextObjectKind::File,
SESSION_PROVIDER,
file.file_ref.clone().unwrap_or_else(|| file.path.clone()),
file.path.clone(),
Some(summary.clone()),
freshness(session.updated_at.to_rfc3339(), file.stale),
1.0,
file.tokens.max(count_tokens(&summary)),
ViewCosts::from_full_tokens(file.tokens.max(count_tokens(&summary))),
Provenance::default(),
metadata,
),
));
}
candidates.sort_by(|(left_time, left), (right_time, right)| {
right_time
.cmp(left_time)
.then_with(|| left.id.as_str().cmp(right.id.as_str()))
});
candidates
.into_iter()
.take(max_candidates)
.map(|(_, candidate)| candidate)
.collect()
}
}
pub(crate) struct LedgerProvider {
project_root: String,
}
impl LedgerProvider {
pub(crate) fn new(project_root: impl Into<String>) -> Self {
Self {
project_root: project_root.into(),
}
}
}
impl CandidateProvider for LedgerProvider {
fn provider_id(&self) -> &str {
LEDGER_PROVIDER
}
fn candidates(&self, ctx: &RetrievalContext) -> Vec<ContextObjectV1> {
let _ = &self.project_root;
ledger_candidates(&ContextLedger::load().entries, ctx.max_candidates)
}
fn side_effect_policy(&self) -> SideEffectPolicy {
SideEffectPolicy::ReadOnly
}
}
#[derive(Clone)]
pub(crate) struct IndexSource {
project_root: String,
index: Arc<OnceLock<Option<Arc<BM25Index>>>>,
hits: Arc<std::sync::Mutex<Option<LastSearch>>>,
}
type LastSearch = (String, usize, Arc<Vec<SearchResult>>);
impl IndexSource {
pub(crate) fn new(project_root: impl Into<String>) -> Self {
Self {
project_root: project_root.into(),
index: Arc::default(),
hits: Arc::default(),
}
}
fn get(&self) -> Option<&Arc<BM25Index>> {
self.index
.get_or_init(|| {
let root = std::path::Path::new(&self.project_root);
let admission = crate::core::context_admission::stores::StoreAdmission::current();
crate::core::bm25_store::get_thread_cache()
.and_then(|cache| crate::core::bm25_cache::peek(&cache, root))
.filter(|index| index.admission_policy.as_deref() == Some(admission.digest()))
.or_else(|| persisted_index(root, admission.digest()))
})
.as_ref()
}
fn search(&self, query: &str, limit: usize) -> Option<Arc<Vec<SearchResult>>> {
let index = self.get()?;
let mut hits = self
.hits
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((cached_query, cached_limit, cached)) = hits.as_ref()
&& cached_query == query
&& *cached_limit == limit
{
return Some(Arc::clone(cached));
}
let found = Arc::new(index.search(query, limit));
*hits = Some((query.to_owned(), limit, Arc::clone(&found)));
Some(found)
}
}
fn persisted_index(root: &std::path::Path, admission: &str) -> Option<Arc<BM25Index>> {
type Key = (
std::path::PathBuf,
crate::core::bm25_cache::IndexFingerprint,
String,
);
static LOADED: std::sync::Mutex<Option<(Key, Arc<BM25Index>)>> = std::sync::Mutex::new(None);
let key: Key = (
root.to_path_buf(),
crate::core::bm25_cache::index_fingerprint(root),
admission.to_owned(),
);
{
let loaded = LOADED
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((cached, index)) = loaded.as_ref()
&& *cached == key
{
return Some(Arc::clone(index));
}
}
let index = Arc::new(BM25Index::load(root)?);
*LOADED
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some((key, Arc::clone(&index)));
Some(index)
}
pub(crate) struct SearchProvider {
source: IndexSource,
}
impl SearchProvider {
pub(crate) fn new(source: IndexSource) -> Self {
Self { source }
}
}
impl CandidateProvider for SearchProvider {
fn provider_id(&self) -> &str {
SEARCH_PROVIDER
}
fn candidates(&self, ctx: &RetrievalContext) -> Vec<ContextObjectV1> {
if ctx.query.trim().is_empty() {
return Vec::new();
}
let Some(hits) = self.source.search(&ctx.query, ctx.max_candidates) else {
return Vec::new();
};
search_candidates(&hits)
}
fn side_effect_policy(&self) -> SideEffectPolicy {
SideEffectPolicy::ReadOnly
}
}
fn search_candidates(hits: &[SearchResult]) -> Vec<ContextObjectV1> {
let top = hits.first().map_or(0.0, |hit| hit.score);
hits.iter()
.map(|hit| {
let reference = format!("{}#L{}-{}", hit.file_path, hit.start_line, hit.end_line);
let tokens = count_tokens(&hit.snippet);
let mut metadata = HashMap::new();
metadata.insert("path".to_string(), hit.file_path.clone());
metadata.insert("symbol".to_string(), hit.symbol_name.clone());
context_object(
ContextItemId::from_provider(SEARCH_PROVIDER, &reference),
ContextObjectKind::SearchChunk,
SEARCH_PROVIDER,
format!("file:{reference}"),
format!("{} ({})", hit.symbol_name, hit.file_path),
Some(hit.snippet.clone()),
freshness(String::new(), false),
if top > 0.0 {
(hit.score / top) as f32
} else {
0.0
},
tokens,
ViewCosts::from_full_tokens(tokens),
Provenance::default(),
metadata,
)
})
.collect()
}
pub(crate) struct GraphProvider {
source: IndexSource,
}
impl GraphProvider {
pub(crate) fn new(source: IndexSource) -> Self {
Self { source }
}
}
impl CandidateProvider for GraphProvider {
fn provider_id(&self) -> &str {
GRAPH_PROVIDER
}
fn candidates(&self, ctx: &RetrievalContext) -> Vec<ContextObjectV1> {
if ctx.query.trim().is_empty() {
return Vec::new();
}
let root = &self.source.project_root;
if !crate::core::property_graph::graph_dir(root)
.join("graph.db")
.is_file()
{
return Vec::new();
}
let Some(hits) = self.source.search(&ctx.query, ctx.max_candidates) else {
return Vec::new();
};
let Ok(graph) = crate::core::property_graph::CodeGraph::open(root) else {
return Vec::new();
};
let mut seeds: Vec<String> = Vec::new();
for hit in hits.iter() {
if !seeds.contains(&hit.file_path) {
seeds.push(hit.file_path.clone());
}
}
let root = std::path::Path::new(root);
graph_candidates(
&seeds,
ctx.max_candidates,
|seed| {
graph
.related_files(seed, GRAPH_NEIGHBOURS_PER_SEED)
.unwrap_or_default()
.into_iter()
.filter(|(path, _)| graph.serves_file(path))
.collect()
},
|path| {
std::fs::metadata(root.join(path)).map_or(0, |meta| {
usize::try_from(meta.len() / 4).unwrap_or(usize::MAX)
})
},
)
}
fn side_effect_policy(&self) -> SideEffectPolicy {
SideEffectPolicy::ReadOnly
}
}
fn graph_candidates(
seeds: &[String],
max_candidates: usize,
related: impl Fn(&str) -> Vec<(String, f64)>,
estimate_tokens: impl Fn(&str) -> usize,
) -> Vec<ContextObjectV1> {
let mut seen: std::collections::HashSet<String> = seeds.iter().cloned().collect();
let mut out = Vec::new();
for seed in seeds {
for (path, weight) in related(seed) {
if out.len() >= max_candidates || !seen.insert(path.clone()) {
continue;
}
let mut metadata = HashMap::new();
metadata.insert("linked_from".to_string(), seed.clone());
metadata.insert("path".to_string(), path.clone());
let tokens = estimate_tokens(&path);
out.push(context_object(
ContextItemId::from_provider(GRAPH_PROVIDER, &path),
ContextObjectKind::File,
GRAPH_PROVIDER,
format!("file:{path}"),
path.clone(),
None,
freshness(String::new(), false),
weight.clamp(0.0, 1.0) as f32,
tokens,
ViewCosts::from_full_tokens(tokens),
Provenance::default(),
metadata,
));
}
}
out
}
pub(crate) fn default_providers(project_root: &str) -> Vec<Box<dyn CandidateProvider>> {
let index = IndexSource::new(project_root);
vec![
Box::new(LedgerProvider::new(project_root)),
Box::new(KnowledgeProvider::new(project_root)),
Box::new(SessionProvider::new(project_root)),
Box::new(SearchProvider::new(index.clone())),
Box::new(GraphProvider::new(index)),
]
}
#[allow(clippy::too_many_arguments)]
fn context_object(
id: ContextItemId,
kind: ContextObjectKind,
source: &str,
content_ref: String,
title: String,
content: Option<String>,
freshness: Freshness,
confidence: f32,
token_estimate: usize,
view_costs: ViewCosts,
provenance: Provenance,
metadata: HashMap<String, String>,
) -> ContextObjectV1 {
ContextObjectV1 {
id,
kind,
source: source.to_string(),
content_ref,
title,
content,
freshness,
confidence,
sensitivity: SensitivityLevel::Internal,
token_estimate,
view_costs,
provenance,
semantic_fingerprint: None,
metadata,
}
}
fn freshness(created_at: String, stale: bool) -> Freshness {
Freshness {
created_at,
ttl_secs: None,
stale,
}
}
fn ledger_candidates(entries: &[LedgerEntry], max_candidates: usize) -> Vec<ContextObjectV1> {
let mut candidates: Vec<(f64, ContextObjectV1)> = entries
.iter()
.filter(|entry| entry.state != Some(ContextState::Excluded))
.map(|entry| {
let mut metadata = HashMap::new();
metadata.insert("mode".to_string(), entry.mode.clone());
metadata.insert("path".to_string(), entry.path.clone());
let tokens = entry.sent_tokens;
(
entry.phi.unwrap_or_default(),
context_object(
entry
.id
.clone()
.unwrap_or_else(|| ContextItemId::from_file(&entry.path)),
ContextObjectKind::File,
LEDGER_PROVIDER,
entry
.source_hash
.clone()
.unwrap_or_else(|| entry.path.clone()),
entry.path.clone(),
None,
freshness(
chrono::DateTime::from_timestamp(entry.timestamp, 0)
.map(|timestamp| timestamp.to_rfc3339())
.unwrap_or_default(),
entry.state == Some(ContextState::Stale),
),
1.0,
tokens,
entry
.view_costs
.clone()
.unwrap_or_else(|| ViewCosts::from_full_tokens(tokens)),
entry.provenance.clone().unwrap_or_default(),
metadata,
),
)
})
.collect();
candidates.sort_by(|(left_phi, left), (right_phi, right)| {
right_phi
.total_cmp(left_phi)
.then_with(|| left.id.as_str().cmp(right.id.as_str()))
});
candidates
.into_iter()
.take(max_candidates)
.map(|(_, candidate)| candidate)
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::context_field::TokenBudget;
fn retrieval_context() -> RetrievalContext {
RetrievalContext {
query: "context kernel provider test".to_string(),
task: None,
project_root: "/__context_kernel_test_project__".to_string(),
budget: TokenBudget {
total: 1_000,
used: 0,
},
max_candidates: 10,
}
}
#[test]
fn provider_ids_are_stable() {
assert_eq!(
KnowledgeProvider::new("/tmp").provider_id(),
KNOWLEDGE_PROVIDER
);
assert_eq!(SessionProvider::new("/tmp").provider_id(), SESSION_PROVIDER);
assert_eq!(LedgerProvider::new("/tmp").provider_id(), LEDGER_PROVIDER);
let index = IndexSource::new("/tmp");
assert_eq!(
SearchProvider::new(index.clone()).provider_id(),
SEARCH_PROVIDER
);
assert_eq!(GraphProvider::new(index).provider_id(), GRAPH_PROVIDER);
}
#[test]
fn provider_side_effect_policies_are_declared() {
assert_eq!(
KnowledgeProvider::new("/tmp").side_effect_policy(),
SideEffectPolicy::MutatesStats
);
for provider in [
SessionProvider::new("/tmp").side_effect_policy(),
LedgerProvider::new("/tmp").side_effect_policy(),
SearchProvider::new(IndexSource::new("/tmp")).side_effect_policy(),
GraphProvider::new(IndexSource::new("/tmp")).side_effect_policy(),
] {
assert_eq!(provider, SideEffectPolicy::ReadOnly);
}
}
#[test]
fn default_providers_include_all_store_wrappers() {
assert_eq!(default_providers("/tmp").len(), 5);
}
#[test]
fn index_sources_carry_provenance_and_bounds() {
use crate::core::bm25_index::ChunkKind;
let hit = |path: &str, score| SearchResult {
chunk_idx: 0,
score,
file_path: path.to_owned(),
symbol_name: "parse".to_owned(),
kind: ChunkKind::Function,
start_line: 3,
end_line: 9,
snippet: "fn parse() {}".to_owned(),
};
let chunks = search_candidates(&[hit("src/a.rs", 4.0), hit("src/b.rs", 1.0)]);
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].source, SEARCH_PROVIDER);
assert_eq!(chunks[0].kind, ContextObjectKind::SearchChunk);
assert_eq!(chunks[0].content_ref, "file:src/a.rs#L3-9");
assert!((chunks[0].confidence - 1.0).abs() < f32::EPSILON);
assert!((chunks[1].confidence - 0.25).abs() < f32::EPSILON);
let seeds = vec!["src/a.rs".to_owned(), "src/b.rs".to_owned()];
let linked = graph_candidates(
&seeds,
2,
|seed| match seed {
"src/a.rs" => vec![("src/b.rs".to_owned(), 0.9), ("src/c.rs".to_owned(), 1.7)],
_ => vec![
("src/c.rs".to_owned(), 0.5),
("src/d.rs".to_owned(), 0.4),
("src/e.rs".to_owned(), 0.3),
],
},
|path| path.len() * 100,
);
let paths: Vec<&str> = linked.iter().map(|object| object.title.as_str()).collect();
assert_eq!(paths, ["src/c.rs", "src/d.rs"]);
assert!(linked.iter().all(|object| object.source == GRAPH_PROVIDER));
assert_eq!(linked[0].token_estimate, 800, "never free to the budget");
assert_ne!(
linked[0].id,
ContextItemId::from_file("src/c.rs"),
"the session/ledger id of the same file stays unambiguous"
);
assert!(
(linked[0].confidence - 1.0).abs() < f32::EPSILON,
"weights are clamped"
);
assert_eq!(linked[0].metadata["linked_from"], "src/a.rs");
}
#[test]
fn index_providers_offer_nothing_without_an_index() {
let ctx = retrieval_context();
let index = IndexSource::new(ctx.project_root.clone());
assert!(
SearchProvider::new(index.clone())
.candidates(&ctx)
.is_empty()
);
assert!(GraphProvider::new(index).candidates(&ctx).is_empty());
assert!(
!crate::core::property_graph::graph_dir(&ctx.project_root)
.join("graph.db")
.exists(),
"planning never creates a graph"
);
}
#[test]
fn ledger_candidates_are_empty_for_an_empty_ledger() {
assert!(ledger_candidates(&ContextLedger::new().entries, 10).is_empty());
}
#[test]
fn ledger_timestamps_use_the_retention_contract() {
use crate::core::context_kernel::policy::ContextPolicy;
let mut ledger = ContextLedger::new();
ledger.record("retention.rs", "full", 10, 10);
ledger.entries[0].timestamp = 0;
let candidates = ledger_candidates(&ledger.entries, 10);
assert_eq!(candidates.len(), 1);
assert_eq!(
candidates[0].freshness.created_at,
"1970-01-01T00:00:00+00:00"
);
let policy = ContextPolicy {
retention_days: Some(1),
..Default::default()
};
let fresh = chrono::DateTime::parse_from_rfc3339("1970-01-01T00:00:01Z").unwrap();
let expired = chrono::DateTime::parse_from_rfc3339("1970-01-02T00:00:00Z").unwrap();
assert!(
policy
.candidate_violation_at(&candidates[0], Some(&fresh))
.is_none()
);
assert_eq!(
policy.candidate_violation_at(&candidates[0], Some(&expired)),
Some("candidate is outside retention window".to_owned())
);
ledger.entries[0].timestamp = i64::MAX;
let invalid = ledger_candidates(&ledger.entries, 10);
assert!(
policy
.candidate_violation_at(&invalid[0], Some(&fresh))
.is_some()
);
}
#[test]
fn knowledge_candidates_are_empty_for_a_missing_project() {
let provider = KnowledgeProvider::new("/__context_kernel_missing_project__");
assert!(provider.candidates(&retrieval_context()).is_empty());
}
fn with_two_findings(root: &str) -> SessionState {
std::fs::write(format!("{root}/pool.rs"), "pub fn lease() {}\n").expect("source");
let mut session = SessionState::new();
session.project_root = Some(root.to_owned());
session.add_finding_for_file(Some("pool.rs"), Some(1), "the pool leaks one lease");
session.add_finding_for_file(None, None, "noted without a file");
session
}
fn offered(session: &SessionState) -> Vec<String> {
let mut contents: Vec<String> = SessionProvider::of_session(session, 10)
.into_iter()
.filter_map(|candidate| candidate.content)
.collect();
contents.sort();
contents
}
#[test]
fn a_finding_whose_source_is_gone_is_not_a_candidate() {
use crate::core::context_admission::stored::lineage_support::{governed, project, scoped};
let _data = governed();
let _community = crate::core::policy::runtime::TestPolicyOverride::set(None);
let (_directory, root) = project();
let session = with_two_findings(&root);
assert_eq!(
scoped(&root, || offered(&session)),
["noted without a file", "the pool leaks one lease"]
);
std::fs::remove_file(format!("{root}/pool.rs")).expect("delete");
assert_eq!(
scoped(&root, || offered(&session)),
["noted without a file"]
);
}
#[test]
fn findings_are_candidates_as_before_in_developer_mode() {
use crate::core::context_admission::stored::lineage_support::project;
let _data = crate::core::data_dir::isolated_data_dir();
let _community = crate::core::policy::runtime::TestPolicyOverride::set(None);
let (_directory, root) = project();
let session = with_two_findings(&root);
std::fs::remove_file(format!("{root}/pool.rs")).expect("delete");
assert_eq!(
offered(&session),
["noted without a file", "the pool leaks one lease"]
);
}
}