_diffctx/parsers/
mod.rs

1mod config_parser;
2mod generic;
3mod markdown;
4mod tree_sitter_strategy;
5
6use std::sync::Arc;
7
8use once_cell::sync::Lazy;
9
10use crate::config::parsers::PARSERS;
11use crate::config::tokenization::TOKENIZATION;
12use crate::types::Fragment;
13
14pub trait FragmentationStrategy: Send + Sync {
15    fn can_handle(&self, path: &str, content: &str) -> bool;
16    fn fragment(&self, path: Arc<str>, content: &str) -> Vec<Fragment>;
17}
18
19static STRATEGIES: Lazy<Vec<Box<dyn FragmentationStrategy>>> = Lazy::new(|| {
20    vec![
21        Box::new(tree_sitter_strategy::TreeSitterStrategy::new()),
22        Box::new(markdown::MarkdownStrategy),
23        Box::new(config_parser::ConfigStrategy),
24        Box::new(generic::GenericStrategy),
25    ]
26});
27
28pub fn fragment_file(path: Arc<str>, content: &str) -> Vec<Fragment> {
29    for strategy in STRATEGIES.iter() {
30        if strategy.can_handle(&path, content) {
31            let fragments = strategy.fragment(Arc::clone(&path), content);
32            if !fragments.is_empty() {
33                return fragments;
34            }
35        }
36    }
37
38    Vec::new()
39}
40
41fn create_snippet(lines: &[&str], start_line: u32, end_line: u32) -> Option<String> {
42    if start_line == 0 || end_line == 0 || start_line > end_line {
43        return None;
44    }
45    let start_idx = (start_line - 1) as usize;
46    let end_idx = end_line as usize;
47    if start_idx >= lines.len() || end_idx > lines.len() {
48        return None;
49    }
50    let mut snippet = lines[start_idx..end_idx].join("\n");
51    if snippet.trim().is_empty() {
52        return None;
53    }
54    if !snippet.ends_with('\n') {
55        snippet.push('\n');
56    }
57    Some(snippet)
58}
59
60fn build_covered_set(covered: &[(u32, u32)]) -> rustc_hash::FxHashSet<u32> {
61    let mut result = rustc_hash::FxHashSet::default();
62    for &(start, end) in covered {
63        for ln in start..=end {
64            result.insert(ln);
65        }
66    }
67    result
68}
69
70fn trim_blank_lines(lines: &[&str], mut start: u32, mut end: u32) -> (u32, u32) {
71    while start <= end
72        && lines
73            .get((start - 1) as usize)
74            .map_or(true, |l| l.trim().is_empty())
75    {
76        start += 1;
77    }
78    while end >= start
79        && lines
80            .get((end - 1) as usize)
81            .map_or(true, |l| l.trim().is_empty())
82    {
83        end -= 1;
84    }
85    (start, end)
86}
87
88fn create_code_gap_fragments(
89    path: Arc<str>,
90    lines: &[&str],
91    covered: &[(u32, u32)],
92) -> Vec<Fragment> {
93    if lines.is_empty() {
94        return Vec::new();
95    }
96
97    let covered_set = build_covered_set(covered);
98    let total = lines.len() as u32;
99
100    let uncovered: Vec<u32> = (1..=total).filter(|ln| !covered_set.contains(ln)).collect();
101    if uncovered.is_empty() {
102        return Vec::new();
103    }
104
105    let mut gaps: Vec<(u32, u32)> = Vec::new();
106    let mut gap_start = uncovered[0];
107    let mut gap_end = uncovered[0];
108    for &ln in &uncovered[1..] {
109        if ln == gap_end + 1 {
110            gap_end = ln;
111        } else {
112            gaps.push((gap_start, gap_end));
113            gap_start = ln;
114            gap_end = ln;
115        }
116    }
117    gaps.push((gap_start, gap_end));
118
119    let mut fragments = Vec::new();
120    for (start, end) in gaps {
121        let (start, end) = trim_blank_lines(lines, start, end);
122        if start > end || end - start + 1 < PARSERS.min_fragment_lines {
123            continue;
124        }
125        if let Some(snippet) = create_snippet(lines, start, end) {
126            let identifiers = crate::types::extract_identifiers(
127                &snippet,
128                TOKENIZATION.fragment_min_identifier_length,
129            );
130            fragments.push(Fragment {
131                id: crate::types::FragmentId::new(Arc::clone(&path), start, end),
132                kind: crate::types::FragmentKind::Chunk,
133                content: Arc::from(snippet),
134                identifiers,
135                token_count: 0,
136                symbol_name: None,
137            });
138        }
139    }
140
141    fragments
142}
_diffctx/parsers/mod.rs

_diffctx/parsers/
mod.rs
