boundgraph_slice/slicer.rs
1//! The contract every slicer meets, and the machinery that makes it hold.
2//!
3//! # Why the guarantee is in the trait rather than in each implementation
4//!
5//! Law II says a step receives only the context it declared, and the cost model
6//! rests on that: `s_max` is what keeps total cost linear in the number of steps
7//! instead of quadratic. A slicer that could exceed its cap would break a bound
8//! the compiler has already promised a reviewer — and it would break it
9//! silently, four slicers deep, in whichever one was written last.
10//!
11//! So [`Slicer::condense`] is a **provided** method. An implementation writes
12//! [`slice`](Slicer::slice) and never sees the cap enforcement, the CAS write,
13//! or the condensate assembly; `condense` does all three around it. Adding a
14//! fifth slicer therefore cannot forget any of them, which is the only way a
15//! guarantee spread across implementations stays true.
16//!
17//! # Over the cap is an error, never a truncation
18//!
19//! [`SliceError::OverCap`] is deliberately not recoverable by chopping the tail
20//! off. A slicer that produced too much has misjudged its own output, and
21//! silently cutting the excess would be exactly the naive truncation the whole
22//! layer exists to avoid — the last thing dropped would be the end of the file,
23//! which is where the interesting part usually is. Failing tells the author
24//! their slicer is wrong; truncating tells the model a lie.
25//!
26//! # Counting tokens without a tokeniser
27//!
28//! Caps are in tokens; nothing here can tokenise. [`TokenCount`] is the seam.
29//! The default, [`Estimate`], is deliberately pessimistic — it assumes text
30//! costs more tokens per byte than it usually does — because the failure modes
31//! are asymmetric: over-estimating wastes a little window, under-estimating
32//! breaks Law II at admission time, after the money is spent. A caller with a
33//! real tokeniser injects it and gets exact behaviour.
34//!
35//! # Determinism
36//!
37//! Slicing is a pure function of `(raw, cap)`. No clock, no randomness, no
38//! iteration over a hashed collection. Replay (US-1006) re-slices recorded raw
39//! output and compares digests, so a slicer that varied between runs would make
40//! a replayed run fail to match for reasons that have nothing to do with the
41//! model.
42
43use std::fmt;
44use std::time::Duration;
45
46use async_trait::async_trait;
47use boundgraph_core::{Condensate, CondensatePayload, Digest, SchemaId, SlicerId};
48use serde::{Deserialize, Serialize};
49
50/// Where raw output is kept so that nothing is lost by slicing.
51///
52/// # Why this is declared here
53///
54/// `ARCHITECTURE.md` §5 lists `Store` among the nine extension points, and
55/// feature 10 will own the durable implementation — the journal references
56/// payloads by CAS digest (US-1005) and so does a large condensate payload
57/// (US-0104). None of that exists yet, and US-0701 needs exactly one thing from
58/// it: somewhere to put the original before the copy is cut down.
59///
60/// So the boundary is declared here, as narrowly as it can be, exactly as the
61/// runtime declares [`LedgerSink`](boundgraph_runtime::LedgerSink) for the log it does
62/// not own. Feature 10's store will satisfy it unchanged.
63#[async_trait]
64pub trait Store: Send + Sync + fmt::Debug {
65 /// Write bytes, returning their content address.
66 ///
67 /// Idempotent by construction: the address *is* the content, so storing the
68 /// same bytes twice is storing them once.
69 ///
70 /// # Errors
71 ///
72 /// [`StoreError`] if the bytes could not be written. The caller must treat
73 /// that as the slice not having happened — a condensate whose `raw` points
74 /// at nothing is worse than no condensate, because it claims an original
75 /// that cannot be produced.
76 async fn put(&self, bytes: &[u8]) -> Result<Digest, StoreError>;
77
78 /// Read bytes back by address.
79 ///
80 /// # Errors
81 ///
82 /// [`StoreError`] if the bytes are not there or could not be read.
83 async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError>;
84}
85
86/// The content-addressed store could not be used.
87#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
88pub enum StoreError {
89 /// Nothing is stored under that address.
90 #[error("nothing is stored at {0}")]
91 NotFound(Digest),
92 /// The store itself failed.
93 #[error("the content store failed: {0}")]
94 Unavailable(String),
95}
96
97/// Turns text into a token count.
98///
99/// See the module documentation for why the default is pessimistic.
100pub trait TokenCount: Send + Sync + fmt::Debug {
101 /// How many tokens `text` is worth.
102 fn count(&self, text: &str) -> usize;
103}
104
105/// The default token estimate: pessimistic, and deterministic.
106///
107/// # How it counts
108///
109/// Three bytes per token, rounded up, plus one token per line. Real byte-level
110/// BPE averages closer to four bytes per token on prose and rather worse on
111/// dense punctuation, which is most of what a slicer handles — code, diffs, and
112/// JSON. The per-line term is there because newlines and the indentation after
113/// them tokenise poorly and a slicer's output is unusually line-dense.
114///
115/// It is an estimate and says so. What matters is the direction of its error:
116/// over-estimating costs a little unused window, under-estimating overruns
117/// `s_max` and breaks Law II after the call has been paid for.
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
119pub struct Estimate;
120
121impl TokenCount for Estimate {
122 fn count(&self, text: &str) -> usize {
123 if text.is_empty() {
124 return 0;
125 }
126 let bytes = text.len();
127 let lines = text.lines().count();
128 bytes.div_ceil(3) + lines
129 }
130}
131
132/// Counts a token per byte.
133///
134/// The only counter that is a genuine upper bound: byte-level BPE never emits
135/// more tokens than there are bytes, because merging is all it does. Far too
136/// conservative for ordinary use — it would spend a 400-token cap on 400 bytes
137/// — but it is the right answer when a cap must not be exceeded under any
138/// tokeniser at all.
139#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
140pub struct ByteCeiling;
141
142impl TokenCount for ByteCeiling {
143 fn count(&self, text: &str) -> usize {
144 text.len()
145 }
146}
147
148/// What running something produced, beyond its output.
149///
150/// Carried separately from the bytes because US-0703 requires exit status and
151/// timing to survive *regardless of the cap*: they are a handful of tokens and
152/// they are the first thing anyone asks about a failed command, so a slicer
153/// that dropped them to fit more log lines would have optimised the wrong way.
154#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
155pub struct CommandOutcome {
156 /// The command line, as invoked.
157 pub command: String,
158 /// The process exit status. `None` where it was killed by a signal or never
159 /// finished.
160 pub exit: Option<i32>,
161 /// How long it ran.
162 pub duration: Duration,
163}
164
165impl CommandOutcome {
166 /// Record a command that finished with a status.
167 pub fn new(command: impl Into<String>, exit: i32, duration: Duration) -> Self {
168 Self {
169 command: command.into(),
170 exit: Some(exit),
171 duration,
172 }
173 }
174
175 /// Whether it succeeded.
176 pub fn succeeded(&self) -> bool {
177 self.exit == Some(0)
178 }
179
180 /// A one-line summary, always retained.
181 pub fn summary(&self) -> String {
182 let status = match self.exit {
183 Some(0) => "ok".to_string(),
184 Some(code) => format!("exit {code}"),
185 None => "killed".to_string(),
186 };
187 format!(
188 "$ {} — {status} in {}ms",
189 self.command,
190 self.duration.as_millis()
191 )
192 }
193}
194
195/// Raw tool output, before slicing.
196#[derive(Debug, Clone, PartialEq, Eq)]
197pub struct Raw {
198 /// The complete, unmodified bytes.
199 pub bytes: Vec<u8>,
200 /// What produced them, where it was a command.
201 pub command: Option<CommandOutcome>,
202 /// A hint at the content, e.g. a file name or language, for slicers that
203 /// can use one. Never load-bearing: a slicer is chosen by the flow.
204 pub hint: Option<String>,
205}
206
207impl Raw {
208 /// Raw bytes with no provenance.
209 pub fn new(bytes: impl Into<Vec<u8>>) -> Self {
210 Self {
211 bytes: bytes.into(),
212 command: None,
213 hint: None,
214 }
215 }
216
217 /// Raw output from a command.
218 #[must_use]
219 pub fn from_command(mut self, outcome: CommandOutcome) -> Self {
220 self.command = Some(outcome);
221 self
222 }
223
224 /// Hint at what this is.
225 #[must_use]
226 pub fn hinting(mut self, hint: impl Into<String>) -> Self {
227 self.hint = Some(hint.into());
228 self
229 }
230
231 /// The bytes as text, replacing anything that is not valid UTF-8.
232 ///
233 /// Lossy rather than failing: tool output is frequently *nearly* UTF-8 —
234 /// a stray byte in a filename, a terminal escape — and refusing to slice a
235 /// build log over one bad byte would be worse than showing a replacement
236 /// character. The original bytes are in the CAS either way.
237 pub fn text(&self) -> std::borrow::Cow<'_, str> {
238 String::from_utf8_lossy(&self.bytes)
239 }
240
241 /// How large the original was.
242 pub fn len(&self) -> usize {
243 self.bytes.len()
244 }
245
246 /// Whether there was any output at all.
247 pub fn is_empty(&self) -> bool {
248 self.bytes.is_empty()
249 }
250}
251
252/// What a slicer produced, before it becomes a [`Condensate`].
253///
254/// The slicer says what it made and whether it dropped anything; `condense`
255/// turns that into the addressed, typed value the rest of the system sees.
256#[derive(Debug, Clone, PartialEq)]
257pub struct Slice {
258 /// The shape of the summary.
259 pub schema: SchemaId,
260 /// The summary itself.
261 pub payload: serde_json::Value,
262 /// Whether information was dropped.
263 ///
264 /// Set honestly. A slicer that under-reports this is telling a downstream
265 /// step it has the whole picture when it does not, and nothing downstream
266 /// can detect the lie — which is why every slicer in this crate has a test
267 /// that a lossless slice reports `false` and a dropping one reports `true`.
268 pub lossy: bool,
269}
270
271impl Slice {
272 /// A slice that dropped nothing.
273 pub fn complete(schema: SchemaId, payload: serde_json::Value) -> Self {
274 Self {
275 schema,
276 payload,
277 lossy: false,
278 }
279 }
280
281 /// A slice that dropped something.
282 pub fn lossy(schema: SchemaId, payload: serde_json::Value) -> Self {
283 Self {
284 schema,
285 payload,
286 lossy: true,
287 }
288 }
289
290 /// The rendered form, which is what the cap is measured against.
291 ///
292 /// A condensate's payload is structured, but what reaches a model is its
293 /// serialisation — so that is what has to fit. Measuring the structure
294 /// instead would let a slicer pass the check and overrun the window.
295 pub fn rendered(&self) -> String {
296 serde_json::to_string(&self.payload).unwrap_or_default()
297 }
298}
299
300/// Slicing failed.
301#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
302#[non_exhaustive]
303pub enum SliceError {
304 /// The slicer produced more than the cap allows.
305 ///
306 /// A bug in the slicer, not a condition to handle: see the module
307 /// documentation for why this is not silently truncated.
308 #[error("slicer {slicer} produced {produced} tokens against a cap of {cap}")]
309 OverCap {
310 /// Which slicer overran.
311 slicer: String,
312 /// What it produced.
313 produced: usize,
314 /// What it was allowed.
315 cap: usize,
316 },
317
318 /// The cap is too small to say anything at all.
319 ///
320 /// Distinct from [`OverCap`](Self::OverCap): the slicer did nothing wrong,
321 /// the step declared a window that cannot hold even a summary. That is a
322 /// flow authoring error and the message says so.
323 #[error("a cap of {cap} tokens cannot hold even a summary (needs at least {minimum})")]
324 CapTooSmall {
325 /// What was allowed.
326 cap: usize,
327 /// The least that would work.
328 minimum: usize,
329 },
330
331 /// The raw output could not be preserved.
332 #[error("the raw output could not be stored: {0}")]
333 Store(#[from] StoreError),
334
335 /// The input was not what this slicer handles.
336 ///
337 /// Carries the reason so a caller can decide to fall back — which is what
338 /// the AST slicer does when a file does not parse (US-0702).
339 #[error("{slicer} cannot slice this input: {detail}")]
340 Unsuitable {
341 /// Which slicer declined.
342 slicer: String,
343 /// Why.
344 detail: String,
345 },
346}
347
348impl SliceError {
349 /// Whether another slicer might succeed where this one did not.
350 ///
351 /// True only for [`Unsuitable`](Self::Unsuitable). An over-cap slicer is
352 /// broken and a too-small cap is a flow error; neither is fixed by trying
353 /// something else, and falling back on them would hide a real problem.
354 pub fn can_fall_back(&self) -> bool {
355 matches!(self, Self::Unsuitable { .. })
356 }
357}
358
359/// Layer 4: turns raw output into a size-capped, typed summary.
360///
361/// Implement [`slice`](Self::slice) and [`id`](Self::id). Do not implement
362/// [`condense`](Self::condense) — it is what enforces the contract, and the
363/// whole design of this module is that an implementation cannot opt out of it.
364#[async_trait]
365pub trait Slicer: Send + Sync + fmt::Debug {
366 /// Which slicer this is, and at what version.
367 ///
368 /// Recorded into every condensate, so re-slicing the same raw output later
369 /// can be checked against the version that originally produced it.
370 fn id(&self) -> SlicerId;
371
372 /// How this slicer counts tokens.
373 ///
374 /// Defaults to [`Estimate`]. Override to inject a real tokeniser.
375 fn counter(&self) -> &dyn TokenCount {
376 &Estimate
377 }
378
379 /// The least a summary from this slicer can cost.
380 ///
381 /// Used to turn an impossible cap into [`SliceError::CapTooSmall`] before
382 /// any work happens, so the author is told their step's window is too small
383 /// rather than handed a mysterious over-cap failure.
384 fn minimum_cap(&self) -> usize {
385 16
386 }
387
388 /// Produce the summary. The part an implementation writes.
389 ///
390 /// Must be a pure function of `(raw, cap)` — see the module documentation
391 /// on determinism.
392 ///
393 /// # Errors
394 ///
395 /// [`SliceError::Unsuitable`] if this slicer does not handle the input.
396 async fn slice(&self, raw: &Raw, cap: usize) -> Result<Slice, SliceError>;
397
398 /// Preserve the original, slice it, and check the result fits.
399 ///
400 /// In that order, and the order matters: the raw bytes reach the store
401 /// *before* anything is dropped, so a slicer that fails part-way through
402 /// still leaves the original recoverable.
403 ///
404 /// # Errors
405 ///
406 /// [`SliceError::Store`] if the original could not be preserved,
407 /// [`SliceError::CapTooSmall`] if the cap cannot hold a summary,
408 /// [`SliceError::OverCap`] if the slicer overran it, or whatever
409 /// [`slice`](Self::slice) returned.
410 async fn condense(
411 &self,
412 raw: &Raw,
413 cap: usize,
414 store: &dyn Store,
415 ) -> Result<Condensate, SliceError> {
416 let minimum = self.minimum_cap();
417 if cap < minimum {
418 return Err(SliceError::CapTooSmall { cap, minimum });
419 }
420
421 // Before slicing, so nothing is lost even if slicing then fails.
422 let digest = store.put(&raw.bytes).await?;
423
424 let slice = self.slice(raw, cap).await?;
425
426 let produced = self.counter().count(&slice.rendered());
427 if produced > cap {
428 return Err(SliceError::OverCap {
429 slicer: self.id().name.clone(),
430 produced,
431 cap,
432 });
433 }
434
435 Ok(Condensate::new(
436 slice.schema,
437 CondensatePayload::inline(&slice.payload).map_err(|e| SliceError::Unsuitable {
438 slicer: self.id().name.clone(),
439 detail: format!("the summary could not be encoded: {e}"),
440 })?,
441 digest,
442 cap,
443 slice.lossy,
444 self.id(),
445 ))
446 }
447}
448
449/// A store that keeps everything in memory.
450///
451/// What the M3 milestone runs on, and a first-class part of the crate for the
452/// same reason the mock reasoner is: a test, a cassette replay, and a policy
453/// check all want a store that needs no filesystem.
454#[derive(Debug, Default)]
455pub struct MemoryStore {
456 blobs: std::sync::Mutex<std::collections::BTreeMap<Digest, Vec<u8>>>,
457}
458
459impl MemoryStore {
460 /// An empty store.
461 pub fn new() -> Self {
462 Self::default()
463 }
464
465 /// How many distinct blobs are held.
466 ///
467 /// Distinct, not written: content addressing means storing the same bytes
468 /// twice stores them once, and this is how that is asserted.
469 pub fn len(&self) -> usize {
470 self.blobs.lock().expect("store lock").len()
471 }
472
473 /// Whether nothing has been stored.
474 pub fn is_empty(&self) -> bool {
475 self.len() == 0
476 }
477}
478
479#[async_trait]
480impl Store for MemoryStore {
481 async fn put(&self, bytes: &[u8]) -> Result<Digest, StoreError> {
482 let digest = Digest::of(bytes);
483 self.blobs
484 .lock()
485 .expect("store lock")
486 .insert(digest, bytes.to_vec());
487 Ok(digest)
488 }
489
490 async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError> {
491 self.blobs
492 .lock()
493 .expect("store lock")
494 .get(digest)
495 .cloned()
496 .ok_or(StoreError::NotFound(*digest))
497 }
498}
499
500#[cfg(test)]
501mod tests {
502 use super::*;
503 use serde_json::json;
504
505 /// A slicer that emits exactly what it is told to, including too much.
506 #[derive(Debug)]
507 struct Fixed {
508 payload: serde_json::Value,
509 lossy: bool,
510 }
511
512 impl Fixed {
513 fn new(payload: serde_json::Value) -> Self {
514 Self {
515 payload,
516 lossy: false,
517 }
518 }
519 }
520
521 #[async_trait]
522 impl Slicer for Fixed {
523 fn id(&self) -> SlicerId {
524 SlicerId::new("fixed", 1)
525 }
526
527 async fn slice(&self, _raw: &Raw, _cap: usize) -> Result<Slice, SliceError> {
528 Ok(Slice {
529 schema: SchemaId::new("test.fixed.v1", 1),
530 payload: self.payload.clone(),
531 lossy: self.lossy,
532 })
533 }
534 }
535
536 fn raw() -> Raw {
537 Raw::new(b"the complete original output".to_vec())
538 }
539
540 // ── the contract ────────────────────────────────────────────────────────
541
542 #[tokio::test]
543 async fn condense_records_the_slicer_id_and_the_cap() {
544 let store = MemoryStore::new();
545 let condensate = Fixed::new(json!("small"))
546 .condense(&raw(), 100, &store)
547 .await
548 .unwrap();
549
550 assert_eq!(condensate.slicer, SlicerId::new("fixed", 1));
551 assert_eq!(condensate.schema, SchemaId::new("test.fixed.v1", 1));
552 assert_eq!(condensate.cap, 100);
553 }
554
555 /// The criterion: raw reaches the CAS, so nothing is lost by slicing.
556 #[tokio::test]
557 async fn the_original_is_preserved_and_addressed_by_the_condensate() {
558 let store = MemoryStore::new();
559 let raw = raw();
560 let condensate = Fixed::new(json!("small"))
561 .condense(&raw, 100, &store)
562 .await
563 .unwrap();
564
565 assert_eq!(condensate.raw, Digest::of(&raw.bytes));
566 assert_eq!(store.get(&condensate.raw).await.unwrap(), raw.bytes);
567 }
568
569 /// And it gets there *before* slicing, so a slicer that fails still leaves
570 /// the original recoverable.
571 #[tokio::test]
572 async fn the_original_survives_a_slicer_that_fails() {
573 #[derive(Debug)]
574 struct Fails;
575
576 #[async_trait]
577 impl Slicer for Fails {
578 fn id(&self) -> SlicerId {
579 SlicerId::new("fails", 1)
580 }
581 async fn slice(&self, _raw: &Raw, _cap: usize) -> Result<Slice, SliceError> {
582 Err(SliceError::Unsuitable {
583 slicer: "fails".into(),
584 detail: "not my kind of input".into(),
585 })
586 }
587 }
588
589 let store = MemoryStore::new();
590 let raw = raw();
591 assert!(Fails.condense(&raw, 100, &store).await.is_err());
592
593 assert_eq!(
594 store.get(&Digest::of(&raw.bytes)).await.unwrap(),
595 raw.bytes,
596 "the original must outlive a failed slice"
597 );
598 }
599
600 /// The criterion the whole module is built around: over the cap is a typed
601 /// error, and it is enforced by the trait rather than by the slicer.
602 #[tokio::test]
603 async fn a_slicer_that_overruns_its_cap_is_refused_not_truncated() {
604 let store = MemoryStore::new();
605 let long = json!("x".repeat(4_000));
606
607 let err = Fixed::new(long)
608 .condense(&raw(), 50, &store)
609 .await
610 .unwrap_err();
611
612 let SliceError::OverCap {
613 slicer,
614 produced,
615 cap,
616 } = err
617 else {
618 panic!("expected OverCap, got {err:?}");
619 };
620 assert_eq!(slicer, "fixed");
621 assert_eq!(cap, 50);
622 assert!(produced > 50);
623 assert!(!err_can_fall_back(produced, cap));
624 }
625
626 fn err_can_fall_back(produced: usize, cap: usize) -> bool {
627 SliceError::OverCap {
628 slicer: "fixed".into(),
629 produced,
630 cap,
631 }
632 .can_fall_back()
633 }
634
635 #[tokio::test]
636 async fn a_cap_too_small_for_any_summary_is_its_own_error() {
637 let store = MemoryStore::new();
638 let err = Fixed::new(json!("small"))
639 .condense(&raw(), 3, &store)
640 .await
641 .unwrap_err();
642
643 assert!(matches!(err, SliceError::CapTooSmall { cap: 3, .. }));
644 assert!(
645 !err.can_fall_back(),
646 "another slicer has the same window problem"
647 );
648 }
649
650 #[tokio::test]
651 async fn only_an_unsuitable_input_invites_a_fallback() {
652 let unsuitable = SliceError::Unsuitable {
653 slicer: "ast".into(),
654 detail: "does not parse".into(),
655 };
656 assert!(unsuitable.can_fall_back());
657 }
658
659 /// Exactly at the cap is allowed; the check is `>`, not `>=`.
660 #[tokio::test]
661 async fn a_slice_exactly_at_the_cap_is_accepted() {
662 let store = MemoryStore::new();
663 let payload = json!("abc");
664 let exact = Estimate.count(&serde_json::to_string(&payload).unwrap());
665
666 assert!(
667 Fixed::new(payload)
668 .condense(&raw(), exact.max(16), &store)
669 .await
670 .is_ok()
671 );
672 }
673
674 #[tokio::test]
675 async fn lossy_is_carried_through_honestly() {
676 let store = MemoryStore::new();
677
678 let clean = Fixed::new(json!("all of it"))
679 .condense(&raw(), 100, &store)
680 .await
681 .unwrap();
682 assert!(!clean.lossy);
683
684 let dropped = Fixed {
685 payload: json!("some of it"),
686 lossy: true,
687 }
688 .condense(&raw(), 100, &store)
689 .await
690 .unwrap();
691 assert!(dropped.lossy);
692 assert!(dropped.needs_raw_for_completeness());
693 }
694
695 /// Replay re-slices recorded output and compares digests, so a slicer that
696 /// varied between runs would break a replay for no reason.
697 #[tokio::test]
698 async fn slicing_the_same_input_twice_gives_the_same_condensate() {
699 let store = MemoryStore::new();
700 let slicer = Fixed::new(json!({ "b": 2, "a": 1 }));
701
702 let first = slicer.condense(&raw(), 100, &store).await.unwrap();
703 for _ in 0..8 {
704 assert_eq!(slicer.condense(&raw(), 100, &store).await.unwrap(), first);
705 }
706 }
707
708 // ── counting ────────────────────────────────────────────────────────────
709
710 #[test]
711 fn the_default_estimate_is_pessimistic_rather_than_accurate() {
712 // Prose really costs about a quarter of its bytes; the estimate says a
713 // third plus a line, and being wrong in that direction is the point.
714 let prose = "the quick brown fox jumps over the lazy dog";
715 assert!(Estimate.count(prose) > prose.len() / 4);
716 assert_eq!(Estimate.count(""), 0);
717 }
718
719 #[test]
720 fn the_estimate_charges_for_lines_because_slicer_output_is_line_dense() {
721 let one_line = "aaaaaaaaaaa";
722 let three_lines = "aaa\naaa\naaa";
723 assert_eq!(
724 one_line.len(),
725 three_lines.len(),
726 "same bytes, different shape"
727 );
728 assert!(Estimate.count(three_lines) > Estimate.count(one_line));
729 }
730
731 /// Byte-level BPE only ever merges, so it cannot emit more tokens than
732 /// there are bytes. That makes this the one counter that is a bound rather
733 /// than an estimate — and, for the same reason, far too conservative to
734 /// use by default.
735 #[test]
736 fn the_byte_ceiling_is_a_genuine_upper_bound() {
737 for text in ["", "a", "hello world", "{\"k\":[1,2,3]}"] {
738 assert_eq!(ByteCeiling.count(text), text.len());
739 assert!(
740 ByteCeiling.count(text) >= Estimate.count(text) || text.len() < 4,
741 "the bound should not sit below the estimate for real inputs"
742 );
743 }
744 }
745
746 // ── the store ───────────────────────────────────────────────────────────
747
748 #[tokio::test]
749 async fn storing_the_same_bytes_twice_stores_them_once() {
750 let store = MemoryStore::new();
751 let a = store.put(b"identical").await.unwrap();
752 let b = store.put(b"identical").await.unwrap();
753
754 assert_eq!(a, b);
755 assert_eq!(store.len(), 1);
756 }
757
758 #[tokio::test]
759 async fn an_address_nothing_was_stored_at_is_a_typed_miss() {
760 let store = MemoryStore::new();
761 let err = store.get(&Digest::of(b"never stored")).await.unwrap_err();
762 assert!(matches!(err, StoreError::NotFound(_)));
763 }
764
765 /// A condensate whose raw points at nothing claims an original that cannot
766 /// be produced, which is worse than no condensate.
767 #[tokio::test]
768 async fn a_store_failure_stops_the_slice() {
769 #[derive(Debug)]
770 struct Broken;
771
772 #[async_trait]
773 impl Store for Broken {
774 async fn put(&self, _bytes: &[u8]) -> Result<Digest, StoreError> {
775 Err(StoreError::Unavailable("disk full".into()))
776 }
777 async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError> {
778 Err(StoreError::NotFound(*digest))
779 }
780 }
781
782 let err = Fixed::new(json!("small"))
783 .condense(&raw(), 100, &Broken)
784 .await
785 .unwrap_err();
786 assert!(matches!(err, SliceError::Store(_)));
787 }
788
789 // ── raw ─────────────────────────────────────────────────────────────────
790
791 #[test]
792 fn raw_text_survives_bytes_that_are_not_utf8() {
793 let raw = Raw::new(vec![b'o', b'k', 0xff, b'!']);
794 assert!(raw.text().contains("ok"));
795 assert_eq!(raw.len(), 4);
796 assert!(!raw.is_empty());
797 }
798
799 #[test]
800 fn a_command_outcome_summarises_status_and_timing() {
801 let ok = CommandOutcome::new("cargo build", 0, Duration::from_millis(1_200));
802 assert!(ok.succeeded());
803 assert!(ok.summary().contains("cargo build"));
804 assert!(ok.summary().contains("ok"));
805 assert!(ok.summary().contains("1200ms"));
806
807 let failed = CommandOutcome::new("cargo test", 101, Duration::from_millis(50));
808 assert!(!failed.succeeded());
809 assert!(failed.summary().contains("exit 101"));
810
811 let killed = CommandOutcome {
812 command: "sleep 100".into(),
813 exit: None,
814 duration: Duration::from_secs(9),
815 };
816 assert!(killed.summary().contains("killed"));
817 }
818}