lean_ctx/core/gateway/postprocess/
index.rs

1//! L3 consolidation (#1095): side-channel — feed downstream output through the
2//! same pipeline as provider data so it becomes searchable (BM25), linked
3//! (property-graph cross-source edges from file references), and remembered
4//! (knowledge). Runs on a background thread and never touches the text returned
5//! to the model, so it cannot perturb output determinism (#498).
6
7use crate::core::bm25_index::ChunkKind;
8use crate::core::consolidation::{self, PrunePrior};
9use crate::core::content_chunk::{ContentChunk, extract_file_references};
10
11/// Resource type recorded for every gateway-proxied tool output, so consolidated
12/// chunks share a stable `gateway://tool_output/…` URI namespace.
13const RESOURCE: &str = "tool_output";
14
15/// Spawn a background job that consolidates one downstream result into the
16/// project's stores. No-op when `project_root` is empty or `text` is blank.
17pub fn spawn(server: String, tool: String, text: String, project_root: String) {
18    if project_root.is_empty() || text.trim().is_empty() {
19        return;
20    }
21    std::thread::spawn(move || run(&server, &tool, &text, &project_root));
22}
23
24/// Synchronous core of [`spawn`] (also the unit-test entry point). Builds an
25/// external content chunk from the tool output and runs the standard
26/// consolidate → persist flow. Best-effort; never panics.
27pub fn run(server: &str, tool: &str, text: &str, project_root: &str) {
28    let chunk = ContentChunk::from_provider(
29        server,
30        RESOURCE,
31        tool,
32        &format!("{server}::{tool}"),
33        ChunkKind::ExternalOther,
34        text.to_string(),
35        extract_file_references(text),
36        None,
37    );
38    let artifacts = consolidation::consolidate(&[chunk]);
39    if artifacts.is_empty() {
40        return;
41    }
42    consolidation::apply_artifacts_to_stores(&artifacts, project_root, &PrunePrior::default());
43}
44
45#[cfg(test)]
46mod tests {
47    use super::*;
48    use crate::core::bm25_index::BM25Index;
49
50    #[test]
51    fn empty_inputs_are_noops() {
52        // Must not panic or write anything for blank scopes/text.
53        run("s", "t", "", "/nonexistent");
54        spawn("s".into(), "t".into(), "x".into(), String::new());
55        spawn("s".into(), "t".into(), String::new(), "/tmp".into());
56    }
57
58    #[test]
59    fn indexed_output_is_searchable() {
60        let _lock = crate::core::data_dir::test_env_lock();
61        let proj = tempfile::tempdir().unwrap();
62        let root = proj.path().to_str().unwrap();
63
64        run(
65            "graphify",
66            "query_graph",
67            "the AuthService handler lives in src/auth/handler.rs and is critical",
68            root,
69        );
70
71        let index = BM25Index::load(proj.path()).expect("index persisted by consolidation");
72        let hits = index.search("AuthService handler", 5);
73        assert!(
74            hits.iter()
75                .any(|h| h.file_path.starts_with("graphify://tool_output/")),
76            "consolidated tool output must be BM25-searchable, got: {:?}",
77            hits.iter().map(|h| &h.file_path).collect::<Vec<_>>()
78        );
79    }
80}
lean_ctx/core/gateway/postprocess/index.rs

lean_ctx/core/gateway/postprocess/
index.rs
