use std::sync::Arc;
use super::{
backend::Backend, block::Headers, normalization::Normalization, split_engine::empty_headers,
ChipError,
};
#[derive(Debug, Clone)]
pub struct ChunkConfig {
pub max_tokens: usize,
pub overlap_tokens: usize,
pub normalization: Normalization,
pub snap_words: bool,
pub include_context: bool,
pub merge_blocks: bool,
pub strict: bool,
}
impl Default for ChunkConfig {
fn default() -> Self {
ChunkConfig {
max_tokens: 512,
overlap_tokens: 50,
normalization: Normalization::Nfc,
snap_words: true,
include_context: true,
merge_blocks: true,
strict: true,
}
}
}
impl ChunkConfig {
pub fn validate(&self) -> Result<(), ChipError> {
if self.max_tokens > 0 && self.overlap_tokens >= self.max_tokens {
return Err(ChipError::InvalidConfig(format!(
"overlap ({}) must be smaller than max_length ({})",
self.overlap_tokens, self.max_tokens
)));
}
Ok(())
}
}
pub fn context_prefix(headers: &[String]) -> String {
format!("[ctx:{}] ", headers.join(" > "))
}
pub(crate) fn fit_context_headers(
headers: &Headers,
max_context_tokens: usize,
backend: &Backend,
) -> Headers {
if headers.is_empty() || max_context_tokens == 0 {
return empty_headers();
}
let all_tokens = backend.count(&context_prefix(headers));
if all_tokens <= max_context_tokens {
return Arc::clone(headers);
}
let mut deepest_tokens = if headers.len() == 1 {
Some(all_tokens)
} else {
None
};
for first in 1..headers.len() {
let candidate = Arc::new(headers[first..].to_vec());
let tokens = backend.count(&context_prefix(&candidate));
if first == headers.len() - 1 {
deepest_tokens = Some(tokens);
}
if tokens <= max_context_tokens {
return candidate;
}
}
let deepest = headers.last().map(String::as_str).unwrap_or("");
if deepest.is_empty() {
return empty_headers();
}
let full_tokens = deepest_tokens.unwrap_or(all_tokens);
let mut keep_chars = if full_tokens <= max_context_tokens {
deepest.chars().count()
} else {
((deepest.chars().count() as u128 * max_context_tokens as u128)
/ full_tokens.max(1) as u128) as usize
};
keep_chars = keep_chars.max(1).min(deepest.chars().count());
loop {
let end = deepest
.char_indices()
.nth(keep_chars)
.map(|(i, _)| i)
.unwrap_or(deepest.len());
if end == 0 {
return empty_headers();
}
let candidate = Arc::new(vec![deepest[..end].to_string()]);
if backend.count(&context_prefix(&candidate)) <= max_context_tokens {
return candidate;
}
if keep_chars <= 1 {
return empty_headers();
}
keep_chars = (keep_chars / 2).max(1);
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Chunk {
pub text: String,
pub start_char: usize,
pub end_char: usize,
pub start_byte: usize,
pub end_byte: usize,
pub token_count: usize,
pub headers: Arc<Vec<String>>,
}
impl Chunk {
pub fn context(&self) -> String {
self.headers.join(" > ")
}
pub fn contextualized_text(&self) -> String {
if self.headers.is_empty() {
self.text.clone()
} else {
format!("{}{}", context_prefix(&self.headers), self.text)
}
}
}