pub const WINDOW: usize = 100;
pub const OVERLAP: usize = 20;
pub const MAX_WINDOW_BYTES: usize = 12_000;
pub const MAX_REQUEST_SOURCE_BYTES: usize = 24_000;
pub const MAX_WINDOWS_PER_REQUEST: usize = 24;
#[derive(Clone)]
pub struct CodeWindow {
pub start: usize,
pub end: usize,
pub snippet: String,
}
pub fn windows(text: &str) -> Vec<CodeWindow> {
let lines: Vec<&str> = text.lines().collect();
let mut result = Vec::new();
let mut start = 0;
while start < lines.len() {
let end = (start + WINDOW).min(lines.len());
push_bounded_windows(&mut result, &lines[start..end], start + 1);
if end == lines.len() {
break;
}
start = end.saturating_sub(OVERLAP);
}
result
}
pub fn push_bounded_windows(result: &mut Vec<CodeWindow>, lines: &[&str], first_line: usize) {
let mut snippet = String::new();
let mut start = first_line;
let mut end = first_line;
let mut has_lines = false;
for (offset, line) in lines.iter().enumerate() {
let line_number = first_line + offset;
if line.len() > MAX_WINDOW_BYTES {
if has_lines {
result.push(CodeWindow {
start,
end,
snippet: std::mem::take(&mut snippet),
});
has_lines = false;
}
for part in utf8_chunks(line, MAX_WINDOW_BYTES) {
result.push(CodeWindow {
start: line_number,
end: line_number,
snippet: part.to_owned(),
});
}
continue;
}
let added = line.len() + usize::from(has_lines);
if has_lines && snippet.len() + added > MAX_WINDOW_BYTES {
result.push(CodeWindow {
start,
end,
snippet: std::mem::take(&mut snippet),
});
has_lines = false;
}
if has_lines {
snippet.push('\n');
} else {
start = line_number;
}
snippet.push_str(line);
end = line_number;
has_lines = true;
}
if has_lines {
result.push(CodeWindow {
start,
end,
snippet,
});
}
}
pub fn utf8_chunks(mut text: &str, max_bytes: usize) -> Vec<&str> {
let mut chunks = Vec::new();
while !text.is_empty() {
let mut end = text.len().min(max_bytes);
while !text.is_char_boundary(end) {
end -= 1;
}
chunks.push(&text[..end]);
text = &text[end..];
}
chunks
}
pub fn window_batches(windows: &[CodeWindow]) -> Vec<&[CodeWindow]> {
let mut batches = Vec::new();
let mut start = 0;
let mut bytes = 0;
for (index, window) in windows.iter().enumerate() {
if index > start
&& (index - start >= MAX_WINDOWS_PER_REQUEST
|| bytes + window.snippet.len() > MAX_REQUEST_SOURCE_BYTES)
{
batches.push(&windows[start..index]);
start = index;
bytes = 0;
}
bytes += window.snippet.len();
}
if start < windows.len() {
batches.push(&windows[start..]);
}
batches
}