lean_ctx/core/
content_chunk.rs

1//! Universal content chunk — the atomic unit of the Context Engine.
2//!
3//! Extends the existing `CodeChunk` (BM25) with a source dimension so that
4//! external data (GitHub issues, Jira tickets, DB schemas, wiki pages) flows
5//! through the same pipeline as code: BM25, embeddings, graph, knowledge.
6//!
7//! Design principles:
8//!   - Backward-compatible: `From<ContentChunk> for CodeChunk` preserves the
9//!     existing BM25 pipeline without changes.
10//!   - Source-aware: `ContentSource` tags where data came from.
11//!   - Reference-carrying: `references` links chunks to code files for
12//!     cross-source graph edges.
13//!
14//! Scientific basis: Neocortical column architecture (Mountcastle) — every
15//! data source is a "column" processing different input through the same
16//! computational template.
17
18use serde::{Deserialize, Serialize};
19
20use super::bm25_index::{ChunkKind, CodeChunk};
21
22/// Where a content chunk originated.
23#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
24#[serde(tag = "type", rename_all = "snake_case")]
25pub enum ContentSource {
26    /// Local filesystem (default, backward-compatible with CodeChunk).
27    #[default]
28    File,
29    /// External data provider (GitHub, Jira, Confluence, etc.).
30    Provider {
31        provider_id: String,
32        resource_type: String,
33    },
34    /// Shell command output.
35    Shell { command: String },
36    /// Knowledge system fact.
37    Knowledge { category: String },
38}
39
40/// A universal content chunk that can represent code, issues, DB schemas,
41/// wiki pages, or any other data source.
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct ContentChunk {
44    pub file_path: String,
45    pub symbol_name: String,
46    pub kind: ChunkKind,
47    pub start_line: usize,
48    pub end_line: usize,
49    pub content: String,
50    #[serde(default)]
51    pub tokens: Vec<String>,
52    pub token_count: usize,
53
54    #[serde(default)]
55    pub source: ContentSource,
56
57    /// URIs or file paths that this chunk references (for cross-source graph edges).
58    #[serde(default)]
59    pub references: Vec<String>,
60
61    /// Provider-specific structured metadata.
62    #[serde(default, skip_serializing_if = "Option::is_none")]
63    pub metadata: Option<serde_json::Value>,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub origin: Option<super::providers::provenance::ProviderOrigin>,
66    /// The untrusted origin of provider text (pz21 §1.6). `None` for a
67    /// project file; a `scheme://` chunk without it is read as unknown.
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub input_origin: Option<super::context_admission::origin::InputOrigin>,
70}
71
72impl ContentChunk {
73    pub fn from_provider(
74        provider_id: &str,
75        resource_type: &str,
76        item_id: &str,
77        title: &str,
78        kind: ChunkKind,
79        content: String,
80        references: Vec<String>,
81        metadata: Option<serde_json::Value>,
82    ) -> Self {
83        let tokens = super::bm25_index::tokenize_for_index(&content);
84        let token_count = tokens.len();
85        Self {
86            file_path: format!("{provider_id}://{resource_type}/{item_id}"),
87            symbol_name: title.to_string(),
88            kind,
89            start_line: 0,
90            end_line: 0,
91            content,
92            tokens,
93            token_count,
94            source: ContentSource::Provider {
95                provider_id: provider_id.to_string(),
96                resource_type: resource_type.to_string(),
97            },
98            references,
99            metadata,
100            origin: None,
101            input_origin: super::context_admission::origin::provider_input_kind(resource_type).map(
102                |kind| {
103                    super::context_admission::origin::InputOrigin::new(
104                        kind,
105                        format!("{provider_id}:{resource_type}/{item_id}"),
106                    )
107                },
108            ),
109        }
110    }
111
112    pub fn is_external(&self) -> bool {
113        !matches!(self.source, ContentSource::File)
114    }
115
116    pub fn provider_id(&self) -> Option<&str> {
117        match &self.source {
118            ContentSource::Provider { provider_id, .. } => Some(provider_id),
119            _ => None,
120        }
121    }
122}
123
124impl From<ContentChunk> for CodeChunk {
125    fn from(c: ContentChunk) -> Self {
126        Self {
127            file_path: c.file_path,
128            symbol_name: c.symbol_name,
129            kind: c.kind,
130            start_line: c.start_line,
131            end_line: c.end_line,
132            content: c.content,
133            tokens: c.tokens,
134            token_count: c.token_count,
135            input_origin: c.input_origin,
136        }
137    }
138}
139
140impl From<CodeChunk> for ContentChunk {
141    fn from(c: CodeChunk) -> Self {
142        Self {
143            file_path: c.file_path,
144            symbol_name: c.symbol_name,
145            kind: c.kind,
146            start_line: c.start_line,
147            end_line: c.end_line,
148            content: c.content,
149            tokens: c.tokens,
150            token_count: c.token_count,
151            source: ContentSource::File,
152            references: Vec::new(),
153            metadata: None,
154            origin: None,
155            input_origin: c.input_origin,
156        }
157    }
158}
159
160// ---------------------------------------------------------------------------
161// Chunk extraction helpers for external data
162// ---------------------------------------------------------------------------
163
164/// Extract file path references from freeform text (issue bodies, PR descriptions).
165/// Looks for patterns like `src/auth.rs`, `lib/handler.ts`, `path/to/file.ext`.
166pub fn extract_file_references(text: &str) -> Vec<String> {
167    static RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
168        regex::Regex::new(r"(?:^|[\s`\(\[])([a-zA-Z0-9_\-./]+\.[a-zA-Z]{1,10})(?:[\s`\)\],:;.]|$)")
169            .expect("file ref regex")
170    });
171
172    let mut refs: Vec<String> = RE
173        .captures_iter(text)
174        .filter_map(|cap| {
175            let path = cap.get(1)?.as_str();
176            if path.contains('/')
177                && !path.starts_with("http")
178                && !path.starts_with("www.")
179                && !path.contains('@')
180            {
181                Some(path.to_string())
182            } else {
183                None
184            }
185        })
186        .collect();
187    refs.sort();
188    refs.dedup();
189    refs
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn content_chunk_to_code_chunk_roundtrip() {
198        let cc = ContentChunk::from_provider(
199            "github",
200            "issues",
201            "123",
202            "Auth token expiry",
203            ChunkKind::Other,
204            "Token expires after 1h".into(),
205            vec!["src/auth.rs".into()],
206            None,
207        );
208
209        assert!(cc.is_external());
210        assert_eq!(cc.provider_id(), Some("github"));
211        assert_eq!(cc.file_path, "github://issues/123");
212
213        let code_chunk: CodeChunk = cc.into();
214        assert_eq!(code_chunk.file_path, "github://issues/123");
215        assert_eq!(code_chunk.symbol_name, "Auth token expiry");
216    }
217
218    #[test]
219    fn code_chunk_to_content_chunk() {
220        let code = CodeChunk {
221            input_origin: None,
222            file_path: "src/main.rs".into(),
223            symbol_name: "main".into(),
224            kind: ChunkKind::Function,
225            start_line: 1,
226            end_line: 10,
227            content: "fn main() {}".into(),
228            tokens: vec!["main".into()],
229            token_count: 1,
230        };
231
232        let cc: ContentChunk = code.into();
233        assert!(!cc.is_external());
234        assert_eq!(cc.source, ContentSource::File);
235        assert!(cc.references.is_empty(), "{:?}", cc.references);
236    }
237
238    #[test]
239    fn extract_file_refs_from_issue_body() {
240        let body = "The bug is in src/auth/handler.rs and affects lib/db.ts.\n\
241                     See also tests/auth_test.rs for the failing test.";
242        let refs = extract_file_references(body);
243        assert!(refs.contains(&"src/auth/handler.rs".to_string()));
244        assert!(refs.contains(&"lib/db.ts".to_string()));
245        assert!(refs.contains(&"tests/auth_test.rs".to_string()));
246    }
247
248    #[test]
249    fn extract_file_refs_ignores_urls() {
250        let body = "See https://github.com/foo/bar.git and www.example.com/page.html";
251        let refs = extract_file_references(body);
252        assert!(refs.is_empty() || !refs.iter().any(|r| r.contains("http")));
253    }
254
255    #[test]
256    fn extract_file_refs_deduplicates() {
257        let body = "Changed src/auth.rs and also src/auth.rs again";
258        let refs = extract_file_references(body);
259        assert_eq!(refs.iter().filter(|r| *r == "src/auth.rs").count(), 1);
260    }
261
262    #[test]
263    fn default_source_is_file() {
264        assert_eq!(ContentSource::default(), ContentSource::File);
265    }
266
267    #[test]
268    fn provider_source_serializes_with_tag() {
269        let src = ContentSource::Provider {
270            provider_id: "jira".into(),
271            resource_type: "issues".into(),
272        };
273        let json = serde_json::to_string(&src).unwrap();
274        assert!(json.contains("\"type\":\"provider\""));
275        assert!(json.contains("\"provider_id\":\"jira\""));
276    }
277}
lean_ctx/core/content_chunk.rs

lean_ctx/core/
content_chunk.rs
