1#![doc = include_str!("../README.md")]
2
3mod certificate;
4mod evidence;
5pub mod ranking;
6mod tokens;
7
8pub use certificate::{
9 ContradictionGroup, CoverageCertificate, FACET_CALLERS, FACET_CONFIG, FACET_DEFAULTS,
10 FACET_DEFINITION, FACET_ERRORS, FACET_EXPANSION_ORDER, FACET_GIT, FACET_GUARDS, FACET_MEMORY,
11 FACET_PUBLIC_API, FACET_SIGNATURES, FACET_TESTS, FacetClaim, is_critical_facet,
12 render_decision_map, render_expansions,
13};
14pub use evidence::{
15 EvidenceDerivation, EvidenceFacet, EvidenceItem, EvidenceLocator, EvidencePriority,
16 EvidenceState, blob_id, evidence_id, packet_id, snapshot_is_stale,
17};
18pub use tokens::{
19 CharDiv4Counter, ChatMessage, ConservativeCounter, RUNTIME_COUNTER, TokenBreakdown,
20 TokenCounter, estimate_tokens,
21};
22
23use std::collections::{HashMap, HashSet};
24use std::fmt::{Display, Formatter};
25
26use serde::{Deserialize, Serialize};
27
28pub const MAX_EVIDENCE_ITEMS: usize = 4_096;
29pub const MAX_EVIDENCE_CHARS: usize = 262_144;
30
31#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
32#[serde(rename_all = "camelCase", deny_unknown_fields)]
33pub struct ContextRequest {
34 pub items: Vec<EvidenceItem>,
35 pub max_tokens: u32,
36 #[serde(default = "enabled")]
48 pub deduplicate: bool,
49}
50
51const fn enabled() -> bool {
52 true
53}
54
55pub const MIN_DEDUPLICATED_LINE_CHARS: usize = 24;
59
60#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
61#[serde(rename_all = "camelCase")]
62pub struct ContextPacket {
63 pub content: String,
64 pub included_ids: Vec<String>,
65 pub omitted_ids: Vec<String>,
66 pub raw_estimated_tokens: u32,
68 pub selected_estimated_tokens: u32,
70 pub omitted_estimated_tokens: u32,
80 pub requires_upstream: bool,
83 #[serde(default)]
88 pub deduplicated_lines: u32,
89 #[serde(default)]
91 pub deduplicated_estimated_tokens: u32,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
94 pub token_breakdown: Option<TokenBreakdown>,
95 #[serde(default, skip_serializing_if = "Option::is_none")]
97 pub packet_id: Option<String>,
98 #[serde(default, skip_serializing_if = "Option::is_none")]
100 pub snapshot_id: Option<String>,
101}
102
103#[derive(Debug, Clone, PartialEq, Eq)]
104pub enum ContextError {
105 EmptyBudget,
106 TooManyItems {
107 count: usize,
108 limit: usize,
109 },
110 EmptyField {
111 index: usize,
112 field: &'static str,
113 },
114 DuplicateId(String),
115 ItemTooLarge {
116 id: String,
117 chars: usize,
118 limit: usize,
119 },
120 CriticalItemExceedsBudget {
121 id: String,
122 tokens: u32,
123 budget: u32,
124 },
125}
126
127impl Display for ContextError {
128 fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
129 match self {
130 Self::EmptyBudget => {
131 formatter.write_str("context token budget must be greater than zero")
132 }
133 Self::TooManyItems { count, limit } => {
134 write!(
135 formatter,
136 "context has {count} evidence items; limit is {limit}"
137 )
138 }
139 Self::EmptyField { index, field } => {
140 write!(formatter, "evidence item {index} has an empty {field}")
141 }
142 Self::DuplicateId(id) => write!(formatter, "duplicate evidence id: {id}"),
143 Self::ItemTooLarge { id, chars, limit } => {
144 write!(
145 formatter,
146 "evidence {id} has {chars} characters; limit is {limit}"
147 )
148 }
149 Self::CriticalItemExceedsBudget { id, tokens, budget } => write!(
150 formatter,
151 "critical evidence {id} needs {tokens} tokens; context budget is {budget}"
152 ),
153 }
154 }
155}
156
157impl std::error::Error for ContextError {}
158
159pub fn compile_context(request: &ContextRequest) -> Result<ContextPacket, ContextError> {
161 compile_context_with(request, &RUNTIME_COUNTER)
162}
163
164pub fn distrust_caller_verified(request: &mut ContextRequest) {
169 for item in &mut request.items {
170 if item.state == EvidenceState::Verified {
171 item.state = EvidenceState::Unverified;
172 if item.derivation == Some(EvidenceDerivation::ExactSource) {
173 item.derivation = Some(EvidenceDerivation::Inferred);
174 }
175 }
176 }
177}
178
179pub fn compile_context_with(
182 request: &ContextRequest,
183 counter: &dyn TokenCounter,
184) -> Result<ContextPacket, ContextError> {
185 validate(request)?;
186 let mut ordered: Vec<_> = request.items.iter().enumerate().collect();
187 ordered.sort_by(|(left_index, left), (right_index, right)| {
188 let state = u8::from(left.state != EvidenceState::Contradictory)
189 .cmp(&u8::from(right.state != EvidenceState::Contradictory));
190 let priority = left.priority.rank().cmp(&right.priority.rank());
191 let relevance = right
194 .relevance
195 .unwrap_or(f64::NEG_INFINITY)
196 .total_cmp(&left.relevance.unwrap_or(f64::NEG_INFINITY));
197 state
198 .then(priority)
199 .then(relevance)
200 .then(left_index.cmp(right_index))
201 });
202
203 let mut content = String::new();
204 let mut included_ids = Vec::new();
205 let mut omitted_ids = Vec::new();
206 let mut seen = HashMap::new();
207 let mut deduplicated_lines = 0_u32;
208 let mut breakdown = TokenBreakdown::for_counter(counter);
209 for (_, item) in &ordered {
210 let full = render_item(item, &item.content);
211 let candidate = counter.count(&full);
212 breakdown.candidate_tokens = breakdown.candidate_tokens.saturating_add(candidate);
213 let (body, removed_lines, _) = if request.deduplicate {
214 deduplicate_body(item, &seen)
215 } else {
216 (item.content.clone(), 0, 0)
217 };
218 let rendered = render_item(item, &body);
219 let delivered = counter.count(&rendered);
220 if breakdown.delivered_tokens.saturating_add(delivered) <= request.max_tokens {
221 content.push_str(&rendered);
222 included_ids.push(item.id.clone());
223 breakdown.selected_before_dedup_tokens = breakdown
224 .selected_before_dedup_tokens
225 .saturating_add(candidate);
226 breakdown.delivered_tokens = breakdown.delivered_tokens.saturating_add(delivered);
227 breakdown.rendering_overhead_tokens = breakdown
228 .rendering_overhead_tokens
229 .saturating_add(delivered.saturating_sub(counter.count(body.trim())));
230 if request.deduplicate {
231 record_substantial_lines(item, &mut seen);
232 deduplicated_lines = deduplicated_lines.saturating_add(removed_lines);
233 }
234 } else if item.priority == EvidencePriority::Critical {
235 return Err(ContextError::CriticalItemExceedsBudget {
236 id: item.id.clone(),
237 tokens: delivered,
238 budget: request.max_tokens,
239 });
240 } else {
241 omitted_ids.push(item.id.clone());
242 breakdown.budget_omitted_tokens =
243 breakdown.budget_omitted_tokens.saturating_add(candidate);
244 }
245 }
246 breakdown.dedup_saved_tokens = breakdown
247 .selected_before_dedup_tokens
248 .saturating_sub(breakdown.delivered_tokens);
249 breakdown.estimated_tokens = breakdown.candidate_tokens;
250 breakdown.wire_tokens = breakdown.delivered_tokens;
251 breakdown.evidence_tokens = breakdown.selected_before_dedup_tokens;
252 breakdown.instruction_tokens = breakdown.rendering_overhead_tokens;
253 breakdown.schema_tokens = 0;
254 breakdown.output_tokens = 0;
255
256 let requires_upstream = request
257 .items
258 .iter()
259 .any(|item| item.state != EvidenceState::Verified);
260 let snapshot = shared_snapshot(&request.items);
261 let delivered_content = content.trim_end().to_owned();
262 let assigned_packet_id = Some(packet_id(&[
263 snapshot.as_deref().unwrap_or_default(),
264 &included_ids.join("\n"),
265 &omitted_ids.join("\n"),
266 &request.max_tokens.to_string(),
267 &delivered_content,
268 ]));
269 Ok(ContextPacket {
270 content: delivered_content,
271 included_ids,
272 omitted_ids,
273 raw_estimated_tokens: breakdown.candidate_tokens,
274 selected_estimated_tokens: breakdown.delivered_tokens,
275 omitted_estimated_tokens: breakdown.budget_omitted_tokens,
276 requires_upstream,
277 deduplicated_lines,
278 deduplicated_estimated_tokens: breakdown.dedup_saved_tokens,
279 token_breakdown: Some(breakdown),
280 snapshot_id: snapshot,
281 packet_id: assigned_packet_id,
282 })
283}
284
285fn shared_snapshot(items: &[EvidenceItem]) -> Option<String> {
286 let mut shared: Option<&str> = None;
287 for item in items {
288 let Some(snapshot) = item
289 .locator
290 .as_ref()
291 .and_then(|locator| locator.snapshot_id.as_deref())
292 else {
293 continue;
294 };
295 match shared {
296 None => shared = Some(snapshot),
297 Some(existing) if existing == snapshot => {}
298 Some(_) => return None,
299 }
300 }
301 shared.map(ToOwned::to_owned)
302}
303
304fn validate(request: &ContextRequest) -> Result<(), ContextError> {
305 if request.max_tokens == 0 {
306 return Err(ContextError::EmptyBudget);
307 }
308 if request.items.len() > MAX_EVIDENCE_ITEMS {
309 return Err(ContextError::TooManyItems {
310 count: request.items.len(),
311 limit: MAX_EVIDENCE_ITEMS,
312 });
313 }
314 let mut ids = HashSet::with_capacity(request.items.len());
315 for (index, item) in request.items.iter().enumerate() {
316 for (field, value) in [
317 ("id", item.id.as_str()),
318 ("source", item.source.as_str()),
319 ("content", item.content.as_str()),
320 ] {
321 if value.trim().is_empty() {
322 return Err(ContextError::EmptyField { index, field });
323 }
324 }
325 if !ids.insert(item.id.as_str()) {
326 return Err(ContextError::DuplicateId(item.id.clone()));
327 }
328 let chars = item.content.chars().count();
329 if chars > MAX_EVIDENCE_CHARS {
330 return Err(ContextError::ItemTooLarge {
331 id: item.id.clone(),
332 chars,
333 limit: MAX_EVIDENCE_CHARS,
334 });
335 }
336 }
337 Ok(())
338}
339
340fn render_item(item: &EvidenceItem, body: &str) -> String {
341 format!(
342 "<evidence id=\"{}\" trust=\"{}\" source=\"{}\">\n<![CDATA[{}]]>\n</evidence>\n\n",
343 xml_escape(&item.id),
344 xml_escape(&item.heading_label()),
345 xml_escape(&item.source),
346 cdata_escape(body.trim())
347 )
348}
349
350fn xml_escape(value: &str) -> String {
351 value
352 .replace('&', "&")
353 .replace('<', "<")
354 .replace('>', ">")
355 .replace('"', """)
356}
357
358fn cdata_escape(value: &str) -> String {
359 value.replace("]]>", "]]]]><![CDATA[>")
360}
361
362#[derive(Debug, Clone, PartialEq, Eq, Hash)]
363struct DedupKey {
364 line: String,
365 state: EvidenceState,
366 derivation: Option<EvidenceDerivation>,
367 path: Option<String>,
368 start_line: Option<u32>,
369 end_line: Option<u32>,
370 snapshot_id: Option<String>,
371 blob_hash: Option<String>,
372}
373
374fn dedup_key(item: &EvidenceItem, line: &str) -> DedupKey {
375 let locator = item.locator.as_ref();
376 DedupKey {
377 line: line.to_owned(),
378 state: item.state,
379 derivation: item.derivation,
380 path: locator.and_then(|locator| locator.path.clone()),
381 start_line: locator.and_then(|locator| locator.start_line),
382 end_line: locator.and_then(|locator| locator.end_line),
383 snapshot_id: locator.and_then(|locator| locator.snapshot_id.clone()),
384 blob_hash: locator.and_then(|locator| locator.blob_hash.clone()),
385 }
386}
387
388fn deduplicate_body(item: &EvidenceItem, seen: &HashMap<DedupKey, String>) -> (String, u32, usize) {
391 let mut kept = Vec::new();
392 let mut dropped = 0_u32;
393 let mut removed_chars = 0_usize;
394 let mut cited = HashSet::new();
395 for line in item.content.lines() {
396 let trimmed = line.trim();
397 if trimmed.chars().count() < MIN_DEDUPLICATED_LINE_CHARS {
398 kept.push(line.to_owned());
399 continue;
400 }
401 if let Some(first) = seen.get(&dedup_key(item, trimmed)) {
402 if cited.insert(first.clone()) {
403 kept.push(format!("same source span as [{first}]"));
404 }
405 dropped = dropped.saturating_add(1);
406 removed_chars = removed_chars.saturating_add(line.chars().count().saturating_add(1));
407 } else {
408 kept.push(line.to_owned());
409 }
410 }
411 if kept.iter().all(|line| line.trim().is_empty()) {
412 return (item.content.clone(), 0, 0);
413 }
414 (kept.join("\n"), dropped, removed_chars)
415}
416
417fn record_substantial_lines(item: &EvidenceItem, seen: &mut HashMap<DedupKey, String>) {
418 for line in item.content.lines() {
419 let trimmed = line.trim();
420 if trimmed.chars().count() >= MIN_DEDUPLICATED_LINE_CHARS {
421 seen.entry(dedup_key(item, trimmed))
422 .or_insert_with(|| item.id.clone());
423 }
424 }
425}
426
427#[cfg(test)]
428#[path = "tests.rs"]
429mod tests;