lean_ctx/core/
content_chunk.rs

1//! Universal content chunk — the atomic unit of the Context Engine.
2//!
3//! Extends the existing `CodeChunk` (BM25) with a source dimension so that
4//! external data (GitHub issues, Jira tickets, DB schemas, wiki pages) flows
5//! through the same pipeline as code: BM25, embeddings, graph, knowledge.
6//!
7//! Design principles:
8//!   - Backward-compatible: `From<ContentChunk> for CodeChunk` preserves the
9//!     existing BM25 pipeline without changes.
10//!   - Source-aware: `ContentSource` tags where data came from.
11//!   - Reference-carrying: `references` links chunks to code files for
12//!     cross-source graph edges.
13//!
14//! Scientific basis: Neocortical column architecture (Mountcastle) — every
15//! data source is a "column" processing different input through the same
16//! computational template.
17
18use serde::{Deserialize, Serialize};
19
20use super::bm25_index::{ChunkKind, CodeChunk};
21
22/// Where a content chunk originated.
23#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
24#[serde(tag = "type", rename_all = "snake_case")]
25pub enum ContentSource {
26    /// Local filesystem (default, backward-compatible with CodeChunk).
27    #[default]
28    File,
29    /// External data provider (GitHub, Jira, Confluence, etc.).
30    Provider {
31        provider_id: String,
32        resource_type: String,
33    },
34    /// Shell command output.
35    Shell { command: String },
36    /// Knowledge system fact.
37    Knowledge { category: String },
38}
39
40/// A universal content chunk that can represent code, issues, DB schemas,
41/// wiki pages, or any other data source.
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct ContentChunk {
44    pub file_path: String,
45    pub symbol_name: String,
46    pub kind: ChunkKind,
47    pub start_line: usize,
48    pub end_line: usize,
49    pub content: String,
50    #[serde(default)]
51    pub tokens: Vec<String>,
52    pub token_count: usize,
53
54    #[serde(default)]
55    pub source: ContentSource,
56
57    /// URIs or file paths that this chunk references (for cross-source graph edges).
58    #[serde(default)]
59    pub references: Vec<String>,
60
61    /// Provider-specific structured metadata.
62    #[serde(default, skip_serializing_if = "Option::is_none")]
63    pub metadata: Option<serde_json::Value>,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub origin: Option<super::providers::provenance::ProviderOrigin>,
66}
67
68impl ContentChunk {
69    pub fn from_provider(
70        provider_id: &str,
71        resource_type: &str,
72        item_id: &str,
73        title: &str,
74        kind: ChunkKind,
75        content: String,
76        references: Vec<String>,
77        metadata: Option<serde_json::Value>,
78    ) -> Self {
79        let tokens = super::bm25_index::tokenize_for_index(&content);
80        let token_count = tokens.len();
81        Self {
82            file_path: format!("{provider_id}://{resource_type}/{item_id}"),
83            symbol_name: title.to_string(),
84            kind,
85            start_line: 0,
86            end_line: 0,
87            content,
88            tokens,
89            token_count,
90            source: ContentSource::Provider {
91                provider_id: provider_id.to_string(),
92                resource_type: resource_type.to_string(),
93            },
94            references,
95            metadata,
96            origin: None,
97        }
98    }
99
100    pub fn is_external(&self) -> bool {
101        !matches!(self.source, ContentSource::File)
102    }
103
104    pub fn provider_id(&self) -> Option<&str> {
105        match &self.source {
106            ContentSource::Provider { provider_id, .. } => Some(provider_id),
107            _ => None,
108        }
109    }
110}
111
112impl From<ContentChunk> for CodeChunk {
113    fn from(c: ContentChunk) -> Self {
114        Self {
115            file_path: c.file_path,
116            symbol_name: c.symbol_name,
117            kind: c.kind,
118            start_line: c.start_line,
119            end_line: c.end_line,
120            content: c.content,
121            tokens: c.tokens,
122            token_count: c.token_count,
123        }
124    }
125}
126
127impl From<CodeChunk> for ContentChunk {
128    fn from(c: CodeChunk) -> Self {
129        Self {
130            file_path: c.file_path,
131            symbol_name: c.symbol_name,
132            kind: c.kind,
133            start_line: c.start_line,
134            end_line: c.end_line,
135            content: c.content,
136            tokens: c.tokens,
137            token_count: c.token_count,
138            source: ContentSource::File,
139            references: Vec::new(),
140            metadata: None,
141            origin: None,
142        }
143    }
144}
145
146// ---------------------------------------------------------------------------
147// Chunk extraction helpers for external data
148// ---------------------------------------------------------------------------
149
150/// Extract file path references from freeform text (issue bodies, PR descriptions).
151/// Looks for patterns like `src/auth.rs`, `lib/handler.ts`, `path/to/file.ext`.
152pub fn extract_file_references(text: &str) -> Vec<String> {
153    static RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
154        regex::Regex::new(r"(?:^|[\s`\(\[])([a-zA-Z0-9_\-./]+\.[a-zA-Z]{1,10})(?:[\s`\)\],:;.]|$)")
155            .expect("file ref regex")
156    });
157
158    let mut refs: Vec<String> = RE
159        .captures_iter(text)
160        .filter_map(|cap| {
161            let path = cap.get(1)?.as_str();
162            if path.contains('/')
163                && !path.starts_with("http")
164                && !path.starts_with("www.")
165                && !path.contains('@')
166            {
167                Some(path.to_string())
168            } else {
169                None
170            }
171        })
172        .collect();
173    refs.sort();
174    refs.dedup();
175    refs
176}
177
178#[cfg(test)]
179mod tests {
180    use super::*;
181
182    #[test]
183    fn content_chunk_to_code_chunk_roundtrip() {
184        let cc = ContentChunk::from_provider(
185            "github",
186            "issues",
187            "123",
188            "Auth token expiry",
189            ChunkKind::Other,
190            "Token expires after 1h".into(),
191            vec!["src/auth.rs".into()],
192            None,
193        );
194
195        assert!(cc.is_external());
196        assert_eq!(cc.provider_id(), Some("github"));
197        assert_eq!(cc.file_path, "github://issues/123");
198
199        let code_chunk: CodeChunk = cc.into();
200        assert_eq!(code_chunk.file_path, "github://issues/123");
201        assert_eq!(code_chunk.symbol_name, "Auth token expiry");
202    }
203
204    #[test]
205    fn code_chunk_to_content_chunk() {
206        let code = CodeChunk {
207            file_path: "src/main.rs".into(),
208            symbol_name: "main".into(),
209            kind: ChunkKind::Function,
210            start_line: 1,
211            end_line: 10,
212            content: "fn main() {}".into(),
213            tokens: vec!["main".into()],
214            token_count: 1,
215        };
216
217        let cc: ContentChunk = code.into();
218        assert!(!cc.is_external());
219        assert_eq!(cc.source, ContentSource::File);
220        assert!(cc.references.is_empty(), "{:?}", cc.references);
221    }
222
223    #[test]
224    fn extract_file_refs_from_issue_body() {
225        let body = "The bug is in src/auth/handler.rs and affects lib/db.ts.\n\
226                     See also tests/auth_test.rs for the failing test.";
227        let refs = extract_file_references(body);
228        assert!(refs.contains(&"src/auth/handler.rs".to_string()));
229        assert!(refs.contains(&"lib/db.ts".to_string()));
230        assert!(refs.contains(&"tests/auth_test.rs".to_string()));
231    }
232
233    #[test]
234    fn extract_file_refs_ignores_urls() {
235        let body = "See https://github.com/foo/bar.git and www.example.com/page.html";
236        let refs = extract_file_references(body);
237        assert!(refs.is_empty() || !refs.iter().any(|r| r.contains("http")));
238    }
239
240    #[test]
241    fn extract_file_refs_deduplicates() {
242        let body = "Changed src/auth.rs and also src/auth.rs again";
243        let refs = extract_file_references(body);
244        assert_eq!(refs.iter().filter(|r| *r == "src/auth.rs").count(), 1);
245    }
246
247    #[test]
248    fn default_source_is_file() {
249        assert_eq!(ContentSource::default(), ContentSource::File);
250    }
251
252    #[test]
253    fn provider_source_serializes_with_tag() {
254        let src = ContentSource::Provider {
255            provider_id: "jira".into(),
256            resource_type: "issues".into(),
257        };
258        let json = serde_json::to_string(&src).unwrap();
259        assert!(json.contains("\"type\":\"provider\""));
260        assert!(json.contains("\"provider_id\":\"jira\""));
261    }
262}
lean_ctx/core/content_chunk.rs

lean_ctx/core/
content_chunk.rs
