use serde_json::{json, Value};
use std::ops::Range;
use std::sync::Arc;
use crate::domain::services::llm::TokenUsage;
const CHUNK_PART_NOTE: &str = "Note: this text is part {i} of {n} of a longer document. \
If the requested information is not present in this part, set the corresponding fields to null.";
const MERGE_INSTRUCTION: &str = "The following JSON array contains partial extraction results \
produced from consecutive parts of a longer document. Combine them into a single result that \
matches the requested schema: choose the best non-null value for each field, concatenate arrays \
for list fields, and output null for fields that are null in every part.\n\nPartial results:\n";
const TERM_STOPWORDS: &[&str] = &[
"id",
"name",
"type",
"value",
"data",
"time",
"url",
"title",
"text",
"key",
"date",
"count",
"index",
"description",
"content",
"items",
"properties",
];
#[derive(Debug, Clone, PartialEq)]
pub struct ChunkingConfig {
pub enabled: bool,
pub model_context_tokens: u32,
pub max_output_tokens: u32,
pub prompt_overhead_tokens: u32,
pub chars_per_token: f64,
pub max_parallel: usize,
pub merge_enabled: bool,
pub term_precheck_enabled: bool,
}
impl Default for ChunkingConfig {
fn default() -> Self {
Self {
enabled: false,
model_context_tokens: 8192,
max_output_tokens: 2048,
prompt_overhead_tokens: 1024,
chars_per_token: 4.0,
max_parallel: 4,
merge_enabled: true,
term_precheck_enabled: true,
}
}
}
impl ChunkingConfig {
pub fn from_settings(settings: &crate::config::settings::LlmChunkingSettings) -> Self {
Self {
enabled: settings.enabled,
model_context_tokens: settings.model_context_tokens,
max_output_tokens: settings.max_output_tokens,
prompt_overhead_tokens: settings.prompt_overhead_tokens,
chars_per_token: settings.chars_per_token,
max_parallel: settings.max_parallel,
merge_enabled: settings.merge_enabled,
term_precheck_enabled: settings.term_precheck_enabled,
}
}
}
pub fn budget_chars(cfg: &ChunkingConfig) -> usize {
let avail = cfg
.model_context_tokens
.saturating_sub(cfg.prompt_overhead_tokens)
.saturating_sub(cfg.max_output_tokens);
let est = (avail as f64) * cfg.chars_per_token / 2.0;
(est as usize).max(1)
}
pub fn plan_chunks(text: &str, cfg: &ChunkingConfig) -> Vec<Range<usize>> {
let budget = budget_chars(cfg);
if text.chars().count() <= budget {
return vec![Range {
start: 0,
end: text.len(),
}];
}
let mut chunks: Vec<Range<usize>> = Vec::new();
let mut chunk_start: usize = 0;
let mut chunk_chars: usize = 0;
for seg in paragraph_segments(text) {
let seg_chars = text[seg.clone()].chars().count();
if seg_chars > budget {
if chunk_chars > 0 {
chunks.push(chunk_start..seg.start);
chunk_chars = 0;
}
emit_oversized_segment(&mut chunks, text, &seg, budget);
chunk_start = seg.end;
continue;
}
if chunk_chars > 0 && chunk_chars + seg_chars > budget {
chunks.push(chunk_start..seg.start);
chunk_start = seg.start;
chunk_chars = 0;
}
if chunk_chars == 0 {
chunk_start = seg.start;
}
chunk_chars += seg_chars;
}
if chunk_start < text.len() {
chunks.push(chunk_start..text.len());
}
chunks
}
fn emit_oversized_segment(
chunks: &mut Vec<Range<usize>>,
text: &str,
seg: &Range<usize>,
budget: usize,
) {
for sub in split_oversized_segment(&text[seg.clone()], budget) {
chunks.push(seg.start + sub.start..seg.start + sub.end);
}
}
fn paragraph_segments(text: &str) -> Vec<Range<usize>> {
let bytes = text.as_bytes();
let mut segs: Vec<Range<usize>> = Vec::new();
let mut start = 0usize;
let mut i = 0usize;
while i < bytes.len() {
let sep_len = if bytes[i..].starts_with(b"\r\n\r\n") {
4
} else if bytes[i..].starts_with(b"\n\n") {
2
} else {
0
};
if sep_len > 0 {
let end = i + sep_len;
segs.push(start..end);
start = end;
i = end;
} else {
i += 1;
}
}
if start < text.len() {
segs.push(start..text.len());
}
segs
}
fn sentence_segments(seg: &str) -> Vec<Range<usize>> {
let mut segs: Vec<Range<usize>> = Vec::new();
let mut start = 0usize;
for (bi, c) in seg.char_indices() {
if matches!(c, '。' | '!' | '?' | '.' | '!' | '?') {
let end = bi + c.len_utf8();
segs.push(start..end);
start = end;
}
}
if start < seg.len() {
segs.push(start..seg.len());
}
segs
}
fn split_oversized_segment(seg: &str, budget: usize) -> Vec<Range<usize>> {
let mut pieces: Vec<Range<usize>> = Vec::new();
let mut acc_start = 0usize;
let mut acc_chars = 0usize;
for s in sentence_segments(seg) {
let s_chars = seg[s.clone()].chars().count();
if s_chars > budget {
if acc_chars > 0 {
pieces.push(acc_start..s.start);
acc_chars = 0;
}
hard_split_sentence(seg, &s, budget, &mut pieces);
acc_start = s.end;
continue;
}
if acc_chars > 0 && acc_chars + s_chars > budget {
pieces.push(acc_start..s.start);
acc_start = s.start;
acc_chars = 0;
}
if acc_chars == 0 {
acc_start = s.start;
}
acc_chars += s_chars;
}
if acc_start < seg.len() && acc_chars > 0 {
pieces.push(acc_start..seg.len());
}
pieces
}
fn hard_split_sentence(seg: &str, s: &Range<usize>, budget: usize, pieces: &mut Vec<Range<usize>>) {
let mut pos = s.start;
let mut used = 0usize;
for (bi, _) in seg[s.start..s.end].char_indices() {
if used == budget {
pieces.push(pos..s.start + bi);
pos = s.start + bi;
used = 0;
}
used += 1;
}
if pos < s.end {
pieces.push(pos..s.end);
}
}
pub fn precheck(schema: &Value, text: &str) -> bool {
let mut terms = Vec::new();
collect_schema_terms(schema, &mut terms);
let meaningful: Vec<&String> = terms
.iter()
.filter(|t| !TERM_STOPWORDS.contains(&t.as_str()))
.collect();
if meaningful.is_empty() {
return true;
}
meaningful
.iter()
.all(|t| contains_ascii_ci(text, t.as_str()))
}
fn contains_ascii_ci(haystack: &str, needle: &str) -> bool {
let n = needle.as_bytes();
if n.is_empty() {
return true;
}
let h = haystack.as_bytes();
if n.len() > h.len() {
return false;
}
h.windows(n.len()).any(|w| w.eq_ignore_ascii_case(n))
}
fn collect_schema_terms(schema: &Value, out: &mut Vec<String>) {
match schema {
Value::Object(map) => {
if let Some(Value::Object(props)) = map.get("properties") {
for (name, prop_schema) in props {
out.extend(split_terms(name));
collect_schema_terms(prop_schema, out);
}
}
if let Some(items) = map.get("items") {
collect_schema_terms(items, out);
}
}
Value::Array(arr) => {
for v in arr {
collect_schema_terms(v, out);
}
}
_ => {}
}
}
fn split_terms(name: &str) -> Vec<String> {
let mut terms = Vec::new();
for seg in name.split(|c: char| !c.is_ascii_alphanumeric()) {
if seg.is_empty() {
continue;
}
split_camel_segment_into(seg, &mut terms);
}
terms
}
fn split_camel_segment_into(seg: &str, terms: &mut Vec<String>) {
let mut cur = String::new();
for c in seg.chars() {
if !cur.is_empty() && c.is_ascii_uppercase() {
terms.push(cur.to_lowercase());
cur = String::new();
}
cur.push(c);
}
if !cur.is_empty() {
terms.push(cur.to_lowercase());
}
}
pub fn merge_results(results: &[Value]) -> Value {
if results.is_empty() {
return json!({});
}
if results.iter().all(Value::is_array) {
return concat_result_arrays(results);
}
let mut merged = serde_json::Map::new();
for r in results {
if let Value::Object(map) = r {
merge_object_fields(&mut merged, map);
}
}
Value::Object(merged)
}
fn concat_result_arrays(results: &[Value]) -> Value {
let mut merged = Vec::new();
for r in results {
if let Value::Array(items) = r {
merged.extend(items.iter().cloned());
}
}
Value::Array(merged)
}
fn merge_object_fields(
merged: &mut serde_json::Map<String, Value>,
map: &serde_json::Map<String, Value>,
) {
for (k, v) in map {
match merged.get(k) {
Some(Value::Array(existing)) if v.is_array() => {
let mut concat = existing.clone();
if let Value::Array(items) = v {
concat.extend(items.iter().cloned());
}
merged.insert(k.clone(), Value::Array(concat));
}
Some(Value::Null) | None => {
merged.insert(k.clone(), v.clone());
}
Some(Value::Array(existing)) if v.is_null() => {
merged.insert(k.clone(), Value::Array(existing.clone()));
}
_ => {
}
}
}
}
pub async fn extract_chunked<E, FutE, M, FutM>(
text: &str,
schema: &Value,
cfg: &ChunkingConfig,
extract_one: E,
merge_call: M,
) -> anyhow::Result<(Value, TokenUsage)>
where
E: Fn(String) -> FutE + Clone + Send + 'static,
FutE: std::future::Future<Output = anyhow::Result<(Value, TokenUsage)>> + Send,
M: Fn(String) -> FutM + Clone + Send + 'static,
FutM: std::future::Future<Output = anyhow::Result<(Value, TokenUsage)>> + Send,
{
if cfg.term_precheck_enabled && !precheck(schema, text) {
log::warn!(
"llm_chunking: term precheck failed — schema terms absent from text, short-circuit"
);
return Ok((json!({}), TokenUsage::default()));
}
if text.chars().count() <= budget_chars(cfg) {
return extract_one(text.to_string()).await;
}
let (chunk_texts, total) = plan_chunk_texts(text, cfg);
log::info!(
"llm_chunking: text_len={} chars, chunks={}, max_parallel={}, merge_enabled={}",
text.chars().count(),
total,
cfg.max_parallel,
cfg.merge_enabled
);
let per_chunk = extract_chunks(chunk_texts, extract_one, cfg.max_parallel).await?;
let (chunk_values, mut total_usage) = collect_chunk_results(per_chunk);
if cfg.merge_enabled && chunk_values.len() > 1 {
if let Some(merged) =
merge_chunk_results(&chunk_values, merge_call, &mut total_usage).await?
{
return Ok((merged, total_usage));
}
}
Ok((merge_results(&chunk_values), total_usage))
}
fn plan_chunk_texts(text: &str, cfg: &ChunkingConfig) -> (Vec<String>, usize) {
let ranges = plan_chunks(text, cfg);
let total = ranges.len();
let chunk_texts = ranges
.iter()
.enumerate()
.map(|(i, r)| {
format!(
"{note}\n\n{chunk}",
note = CHUNK_PART_NOTE
.replace("{i}", &(i + 1).to_string())
.replace("{n}", &total.to_string()),
chunk = &text[r.clone()],
)
})
.collect();
(chunk_texts, total)
}
async fn extract_chunks<E, FutE>(
chunk_texts: Vec<String>,
extract_one: E,
max_parallel: usize,
) -> anyhow::Result<Vec<(Value, TokenUsage)>>
where
E: Fn(String) -> FutE + Clone + Send + 'static,
FutE: std::future::Future<Output = anyhow::Result<(Value, TokenUsage)>> + Send,
{
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_parallel.max(1)));
let mut join_set: tokio::task::JoinSet<(usize, anyhow::Result<(Value, TokenUsage)>)> =
tokio::task::JoinSet::new();
let total = chunk_texts.len();
for (idx, chunk_text) in chunk_texts.into_iter().enumerate() {
let permit_sem = semaphore.clone();
let extract_one = extract_one.clone();
join_set.spawn(async move {
let _permit = permit_sem.acquire_owned().await;
(idx, extract_one(chunk_text).await)
});
}
let mut per_chunk: Vec<Option<(Value, TokenUsage)>> = (0..total).map(|_| None).collect();
while let Some(joined) = join_set.join_next().await {
let (idx, result) = joined.map_err(|e| anyhow::anyhow!("chunk task panicked: {e}"))?;
let (value, usage) = result?;
per_chunk[idx] = Some((value, usage));
}
Ok(per_chunk.into_iter().flatten().collect())
}
fn collect_chunk_results(per_chunk: Vec<(Value, TokenUsage)>) -> (Vec<Value>, TokenUsage) {
let mut total_usage = TokenUsage::default();
let mut chunk_values: Vec<Value> = Vec::with_capacity(per_chunk.len());
for (value, usage) in per_chunk {
total_usage.prompt_tokens += usage.prompt_tokens;
total_usage.completion_tokens += usage.completion_tokens;
total_usage.total_tokens += usage.total_tokens;
chunk_values.push(value);
}
(chunk_values, total_usage)
}
async fn merge_chunk_results<M, FutM>(
chunk_values: &[Value],
merge_call: M,
total_usage: &mut TokenUsage,
) -> anyhow::Result<Option<Value>>
where
M: Fn(String) -> FutM,
FutM: std::future::Future<Output = anyhow::Result<(Value, TokenUsage)>> + Send,
{
let payload = format!(
"{instruction}{results}",
instruction = MERGE_INSTRUCTION,
results = serde_json::to_string(chunk_values)?,
);
match merge_call(payload).await {
Ok((merged, usage)) => {
total_usage.prompt_tokens += usage.prompt_tokens;
total_usage.completion_tokens += usage.completion_tokens;
total_usage.total_tokens += usage.total_tokens;
Ok(Some(merged))
}
Err(e) => {
log::warn!("llm_chunking: merge call failed ({e}); falling back to heuristic merge");
Ok(None)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
use std::sync::Mutex;
#[test]
fn budget_chars_default_config_arithmetic() {
let cfg = ChunkingConfig::default();
assert_eq!(budget_chars(&cfg), 10240);
}
#[test]
fn budget_chars_floors_at_one() {
let cfg = ChunkingConfig {
model_context_tokens: 100,
prompt_overhead_tokens: 200,
max_output_tokens: 300,
..ChunkingConfig::default()
};
assert_eq!(budget_chars(&cfg), 1);
}
#[test]
fn plan_chunks_single_block_when_under_budget() {
let cfg = cfg_with_budget(100);
let text = "short text".to_string();
let chunks = plan_chunks(&text, &cfg);
assert_eq!(chunks, vec![0..text.len()]);
}
#[test]
fn plan_chunks_chunk_count_matches_ceil_and_tiles_text() {
let mut text = String::new();
for i in 0..40 {
text.push_str(&format!("para-{:02}-x\n\n", i));
}
let text = text.trim_end_matches('\n').to_string();
let cfg = cfg_with_budget(100);
let chunks = plan_chunks(&text, &cfg);
assert_eq!(
chunks.len(),
text.chars().count().div_ceil(100),
"块数应为 ceil(len/budget)"
);
assert_tiles(text.as_str(), &chunks);
}
#[test]
fn plan_chunks_prefers_paragraph_boundaries() {
let cfg = cfg_with_budget(30);
let text = "aa bb cc dd\n\nee ff gg hh\n\nii jj kk ll mm nn";
let chunks = plan_chunks(text, &cfg);
assert_tiles(text, &chunks);
for (i, r) in chunks.iter().enumerate() {
let seg = &text[r.clone()];
let is_last = i == chunks.len() - 1;
assert!(
is_last || seg.ends_with("\n\n"),
"块应切在段落边界: {seg:?}"
);
}
}
#[test]
fn plan_chunks_handles_crlf_paragraphs() {
let cfg = cfg_with_budget(20);
let text = "aa bb cc\r\n\r\ndd ee ff\r\n\r\ngg hh ii";
let chunks = plan_chunks(text, &cfg);
assert_tiles(text, &chunks);
}
#[test]
fn plan_chunks_falls_back_to_sentence_boundaries() {
let cfg = cfg_with_budget(25);
let text = "First sentence here. Second sentence follows. Third one is longer!";
let chunks = plan_chunks(text, &cfg);
assert!(chunks.len() > 1, "超预算应多块");
assert_tiles(text, &chunks);
let last = chunks.last().unwrap();
assert!(text[last.clone()].ends_with('!'));
}
#[test]
fn plan_chunks_hard_cuts_oversized_sentence() {
let cfg = cfg_with_budget(10);
let text = "abcdefghij".to_string();
let text = format!("{text}{text}{text}"); let chunks = plan_chunks(&text, &cfg);
assert_eq!(chunks.len(), 3);
assert_tiles(&text, &chunks);
for r in &chunks {
assert!(text[r.clone()].chars().count() <= 10);
}
}
#[test]
fn plan_chunks_hard_cut_respects_utf8_boundaries() {
let cfg = cfg_with_budget(3);
let text = "一二三四五六七八九十".to_string(); let chunks = plan_chunks(&text, &cfg);
assert_tiles(&text, &chunks);
for r in &chunks {
assert!(text[r.clone()].chars().count() <= 3);
}
}
#[test]
fn plan_chunks_mixed_paragraph_with_oversized_sentence() {
let cfg = cfg_with_budget(8);
let text = format!("short.\n\n{}", "x".repeat(30));
let chunks = plan_chunks(&text, &cfg);
assert_tiles(&text, &chunks);
for r in &chunks {
assert!(text[r.clone()].chars().count() <= 8, "块 {:?} 超预算", r);
}
}
#[test]
fn default_config_is_disabled_with_spec_defaults() {
let cfg = ChunkingConfig::default();
assert!(!cfg.enabled);
assert_eq!(cfg.model_context_tokens, 8192);
assert_eq!(cfg.max_output_tokens, 2048);
assert_eq!(cfg.prompt_overhead_tokens, 1024);
assert!((cfg.chars_per_token - 4.0).abs() < f64::EPSILON);
assert_eq!(cfg.max_parallel, 4);
assert!(cfg.merge_enabled);
assert!(cfg.term_precheck_enabled);
}
#[test]
fn precheck_passes_when_terms_present_in_text() {
let schema = json!({
"type": "object",
"properties": { "product_price": { "type": "number" } }
});
assert!(precheck(&schema, "The product price is 42 dollars."));
}
#[test]
fn precheck_splits_camel_case_terms() {
let schema = json!({
"type": "object",
"properties": { "productPrice": { "type": "number" } }
});
assert!(precheck(&schema, "the product price"));
}
#[test]
fn precheck_fails_when_terms_absent_from_text() {
let schema = json!({
"type": "object",
"properties": { "qwzx": { "type": "string" } }
});
assert!(!precheck(&schema, "nothing relevant here at all."));
}
#[test]
fn precheck_collects_nested_property_names() {
let schema = json!({
"type": "object",
"properties": {
"author": {
"type": "object",
"properties": { "biography": { "type": "string" } }
}
}
});
assert!(precheck(&schema, "author biography mentioned"));
assert!(!precheck(&schema, "neither word appears here"));
}
#[test]
fn precheck_all_stopwords_skips_verdict() {
let schema = json!({
"type": "object",
"properties": {
"id": { "type": "string" },
"name": { "type": "string" }
}
});
assert!(precheck(&schema, "completely unrelated text"));
}
#[test]
fn precheck_schema_without_properties_passes() {
let schema = json!({ "type": "array", "description": "list of items" });
assert!(precheck(&schema, "any text"));
}
#[test]
fn merge_results_takes_first_non_null_per_field() {
let results = vec![
json!({"title": "from chunk 1", "x": null}),
json!({"title": "from chunk 2"}),
json!({"title": null, "y": 2}),
];
let merged = merge_results(&results);
assert_eq!(merged["title"], "from chunk 1");
assert_eq!(merged["x"], Value::Null);
assert_eq!(merged["y"], 2);
}
#[test]
fn merge_results_concatenates_array_fields() {
let results = vec![json!({"items": [1, 2]}), json!({"items": [3]})];
let merged = merge_results(&results);
assert_eq!(merged["items"], json!([1, 2, 3]));
}
#[test]
fn merge_results_concatenates_root_arrays() {
let results = vec![json!([{"a": 1}]), json!([{"b": 2}])];
let merged = merge_results(&results);
assert_eq!(merged, json!([{"a": 1}, {"b": 2}]));
}
#[test]
fn merge_results_empty_input_yields_empty_object() {
assert_eq!(merge_results(&[]), json!({}));
}
#[tokio::test]
async fn extract_chunked_single_shot_when_under_budget() {
let cfg = cfg_with_budget(1000);
let calls = Arc::new(Mutex::new(Vec::<String>::new()));
let calls_c = calls.clone();
let (value, usage) = extract_chunked(
"plain short text about products",
&json!({"type": "object", "properties": {"product": {"type": "string"}}}),
&cfg,
move |text: String| {
let calls = calls_c.clone();
async move {
calls.lock().unwrap().push(text);
Ok((json!({"product": "p"}), usage_of(10, 5)))
}
},
move |_payload: String| async { anyhow::bail!("merge should not be called") },
)
.await
.unwrap();
assert_eq!(value, json!({"product": "p"}));
assert_eq!(usage.total_tokens, 15);
let calls = calls.lock().unwrap();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0], "plain short text about products");
}
#[tokio::test]
async fn extract_chunked_precheck_short_circuits() {
let cfg = cfg_with_budget(1000);
let calls = Arc::new(AtomicUsize::new(0));
let calls_c = calls.clone();
let (value, usage) = extract_chunked(
"nothing relevant here",
&json!({"type": "object", "properties": {"qwzx": {"type": "string"}}}),
&cfg,
move |_text: String| {
let calls = calls_c.clone();
async move {
calls.fetch_add(1, AtomicOrdering::SeqCst);
Ok((json!({}), usage_of(1, 1)))
}
},
move |_payload: String| async { anyhow::bail!("merge should not be called") },
)
.await
.unwrap();
assert_eq!(value, json!({}));
assert_eq!(usage.total_tokens, 0);
assert_eq!(calls.load(AtomicOrdering::SeqCst), 0, "provider 不应被调用");
}
#[tokio::test]
async fn extract_chunked_precheck_disabled_proceeds() {
let cfg = ChunkingConfig {
term_precheck_enabled: false,
..cfg_with_budget(1000)
};
let calls = Arc::new(AtomicUsize::new(0));
let calls_c = calls.clone();
let _ = extract_chunked(
"nothing relevant here",
&json!({"type": "object", "properties": {"qwzx": {"type": "string"}}}),
&cfg,
move |_text: String| {
let calls = calls_c.clone();
async move {
calls.fetch_add(1, AtomicOrdering::SeqCst);
Ok((json!({"qwzx": "found"}), usage_of(1, 1)))
}
},
move |_payload: String| async { anyhow::bail!("merge should not be called") },
)
.await
.unwrap();
assert_eq!(calls.load(AtomicOrdering::SeqCst), 1);
}
#[tokio::test]
async fn extract_chunked_runs_parallel_within_max_parallel() {
let cfg = ChunkingConfig {
max_parallel: 2,
..cfg_with_budget(20)
};
let text = "aaaa bbbb cccc dd\neeee ffff gggg hh\niiii jjjj kkkk ll\nmmmm nnnn oooo pp"
.to_string();
let schema = json!({"type": "object", "properties": {"text": {"type": "string"}}});
let calls = Arc::new(AtomicUsize::new(0));
let inflight = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let texts = Arc::new(Mutex::new(Vec::<String>::new()));
let (calls_c, inflight_c, peak_c, texts_c) =
(calls.clone(), inflight.clone(), peak.clone(), texts.clone());
let (value, usage) = extract_chunked(
&text,
&schema,
&cfg,
move |chunk: String| {
let (calls, inflight, peak, texts) = (
calls_c.clone(),
inflight_c.clone(),
peak_c.clone(),
texts_c.clone(),
);
async move {
calls.fetch_add(1, AtomicOrdering::SeqCst);
let now = inflight.fetch_add(1, AtomicOrdering::SeqCst) + 1;
peak.fetch_max(now, AtomicOrdering::SeqCst);
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
inflight.fetch_sub(1, AtomicOrdering::SeqCst);
texts.lock().unwrap().push(chunk);
Ok((json!({"text": "p"}), usage_of(7, 3)))
}
},
move |_payload: String| async { anyhow::bail!("merge should not be called") },
)
.await
.unwrap();
assert_eq!(calls.load(AtomicOrdering::SeqCst), 4, "4 块各提取一次");
assert!(
peak.load(AtomicOrdering::SeqCst) <= 2,
"并发峰值必须 ≤ max_parallel"
);
assert!(
peak.load(AtomicOrdering::SeqCst) >= 2,
"JoinSet 应真实并行(峰值 > 1)"
);
let texts = texts.lock().unwrap();
for (i, t) in texts.iter().enumerate() {
assert!(t.contains("part"), "块 {i} 应带 part 注记");
}
assert_eq!(value, json!({"text": "p"}));
assert_eq!(usage.prompt_tokens, 28);
assert_eq!(usage.completion_tokens, 12);
assert_eq!(usage.total_tokens, 40);
}
#[tokio::test]
async fn extract_chunked_merge_prompt_combines_chunk_results() {
let cfg = ChunkingConfig {
merge_enabled: true,
..cfg_with_budget(20)
};
let text = "alpha beta gamma\n\nzeta eta theta\n\niota kappa lamb".to_string();
let merge_payloads = Arc::new(Mutex::new(Vec::<String>::new()));
let merge_payloads_c = merge_payloads.clone();
let (value, usage) = extract_chunked(
&text,
&json!({"type": "object", "properties": {"title": {"type": "string"}}}),
&cfg,
move |chunk: String| {
let title = if chunk.contains("zeta") {
"title-zeta"
} else {
"title-other"
};
async move { Ok((json!({ "title": title }), usage_of(10, 2))) }
},
move |payload: String| {
let payloads = merge_payloads_c.clone();
async move {
payloads.lock().unwrap().push(payload);
Ok((json!({"title": "merged-best"}), usage_of(30, 5)))
}
},
)
.await
.unwrap();
assert_eq!(value, json!({"title": "merged-best"}));
let payloads = merge_payloads.lock().unwrap();
assert_eq!(payloads.len(), 1, "merge 只调一次");
let payload = &payloads[0];
assert!(payload.contains("Combine them into a single result"));
assert!(payload.contains("\"title\""), "应含块结果 JSON 数组字段");
assert!(
payload.contains("title-zeta") && payload.contains("title-other"),
"应含各块结果: {payload}"
);
assert_eq!(usage.prompt_tokens, 30 + 30);
assert_eq!(usage.completion_tokens, 6 + 5);
assert_eq!(usage.total_tokens, 36 + 35);
}
#[tokio::test]
async fn extract_chunked_merge_disabled_uses_heuristic_merge() {
let cfg = ChunkingConfig {
merge_enabled: false,
..cfg_with_budget(20)
};
let text = "alpha beta gamma\n\nzeta eta theta\n\niota kappa lamb".to_string();
let (value, usage) = extract_chunked(
&text,
&json!({"type": "object", "properties": {"title": {"type": "string"}}}),
&cfg,
move |chunk: String| {
let title = if chunk.contains("zeta") {
"winner-zeta"
} else {
""
};
async move {
Ok((
if title.is_empty() {
json!({"title": null})
} else {
json!({"title": title})
},
usage_of(10, 2),
))
}
},
move |_payload: String| async { anyhow::bail!("merge must not be called") },
)
.await
.unwrap();
assert_eq!(value, json!({"title": "winner-zeta"}), "取首个非 null");
assert_eq!(usage.total_tokens, 36, "仅 Σ块用量");
}
#[tokio::test]
async fn extract_chunked_merge_failure_falls_back_to_heuristic() {
let cfg = cfg_with_budget(20);
let text = "alpha beta gamma\n\nzeta eta theta\n\niota kappa lamb".to_string();
let (value, usage) = extract_chunked(
&text,
&json!({"type": "object", "properties": {"title": {"type": "string"}}}),
&cfg,
move |_chunk: String| async { Ok((json!({"title": "chunk-title"}), usage_of(10, 2))) },
move |_payload: String| async { anyhow::bail!("merge provider down") },
)
.await
.unwrap();
assert_eq!(
value,
json!({"title": "chunk-title"}),
"启发式兜底取首个非 null"
);
assert_eq!(usage.total_tokens, 36);
}
#[tokio::test]
async fn extract_chunked_chunk_failure_propagates_error() {
let cfg = cfg_with_budget(20);
let text =
"alpha beta gamma delta\n\nepsilon zeta eta theta\n\niota kappa lambda mu".to_string();
let result = extract_chunked(
&text,
&json!({"type": "object", "properties": {"title": {"type": "string"}}}),
&cfg,
move |chunk: String| async move {
if chunk.contains("epsilon") {
anyhow::bail!("provider 429")
}
Ok((json!({"title": "ok"}), usage_of(1, 1)))
},
move |_payload: String| async { anyhow::bail!("merge must not be called") },
)
.await;
let err = result.expect_err("块失败必须整体 Err");
assert!(err.to_string().contains("429"), "错误信息保留: {err}");
}
#[tokio::test]
async fn extract_chunked_empty_usage_default() {
let cfg = cfg_with_budget(1000);
let (_, usage) = extract_chunked(
"tiny text",
&json!({"type": "object", "properties": {"product": {"type": "string"}}}),
&cfg,
move |_text: String| async { Ok((json!({}), TokenUsage::default())) },
move |_payload: String| async { anyhow::bail!("merge must not be called") },
)
.await
.unwrap();
assert_eq!(usage, TokenUsage::default());
}
fn usage_of(prompt: u32, completion: u32) -> TokenUsage {
TokenUsage {
prompt_tokens: prompt,
completion_tokens: completion,
total_tokens: prompt + completion,
}
}
fn cfg_with_budget(n: usize) -> ChunkingConfig {
ChunkingConfig {
enabled: true,
max_output_tokens: (8192 - 1024 - (n / 2)) as u32,
..ChunkingConfig::default()
}
}
fn assert_tiles(text: &str, chunks: &[Range<usize>]) {
assert!(!chunks.is_empty(), "至少一块");
let mut cursor = 0usize;
for r in chunks {
assert_eq!(r.start, cursor, "块无缝衔接");
assert!(r.end >= r.start);
cursor = r.end;
}
assert_eq!(cursor, text.len(), "块拼接覆盖全文");
let joined: String = chunks.iter().map(|r| &text[r.clone()]).collect();
assert_eq!(joined, text);
}
}