use std::sync::Arc;
use crate::config::parsers::PARSERS;
use crate::config::tokenization::TOKENIZATION;
use crate::types::{Fragment, FragmentId, FragmentKind, extract_identifiers};
use super::FragmentationStrategy;
pub struct GenericStrategy;
impl FragmentationStrategy for GenericStrategy {
fn can_handle(&self, _path: &str, _content: &str) -> bool {
true
}
fn fragment(&self, path: Arc<str>, content: &str) -> Vec<Fragment> {
if content.trim().is_empty() {
return Vec::new();
}
let lines: Vec<&str> = content.split('\n').collect();
if lines.is_empty() {
return Vec::new();
}
let total = lines.len();
let mut fragments: Vec<Fragment> = Vec::new();
let mut start_idx: usize = 0;
while start_idx < total {
let end_idx = (start_idx + PARSERS.generic_max_lines - 1).min(total - 1);
let start_line = start_idx as u32 + 1;
let end_line = end_idx as u32 + 1;
let mut snippet = lines[start_idx..=end_idx].join("\n");
if !snippet.ends_with('\n') {
snippet.push('\n');
}
let identifiers =
extract_identifiers(&snippet, TOKENIZATION.fragment_min_identifier_length);
fragments.push(Fragment {
id: FragmentId::new(Arc::clone(&path), start_line, end_line),
kind: FragmentKind::Chunk,
content: Arc::from(snippet),
identifiers,
token_count: 0,
symbol_name: None,
});
start_idx = end_idx + 1;
}
fragments
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_file_yields_no_fragments() {
assert!(
GenericStrategy
.fragment(Arc::from("empty.bin"), "")
.is_empty()
);
}
#[test]
fn blank_lines_only_yields_no_fragments() {
assert!(
GenericStrategy
.fragment(Arc::from("blank.bin"), "\n\n\n")
.is_empty()
);
}
#[test]
fn whitespace_only_yields_no_fragments() {
assert!(
GenericStrategy
.fragment(Arc::from("whitespace.bin"), " ")
.is_empty()
);
}
#[test]
fn thousand_line_file_splits_into_contiguous_non_overlapping_chunks() {
let content: String = (1..=1000)
.map(|i| format!("line {i}"))
.collect::<Vec<_>>()
.join("\n");
let fragments = GenericStrategy.fragment(Arc::from("big.txt"), &content);
let expected_count = 1000usize.div_ceil(PARSERS.generic_max_lines);
assert_eq!(fragments.len(), expected_count);
let mut next_expected_start = 1u32;
for fragment in &fragments {
assert_eq!(fragment.id.start_line, next_expected_start);
assert!(fragment.id.start_line <= fragment.id.end_line);
next_expected_start = fragment.id.end_line + 1;
}
assert_eq!(fragments.last().unwrap().id.end_line, 1000);
}
}