Skip to main content

boundgraph_slice/
slicer.rs

1//! The contract every slicer meets, and the machinery that makes it hold.
2//!
3//! # Why the guarantee is in the trait rather than in each implementation
4//!
5//! Law II says a step receives only the context it declared, and the cost model
6//! rests on that: `s_max` is what keeps total cost linear in the number of steps
7//! instead of quadratic. A slicer that could exceed its cap would break a bound
8//! the compiler has already promised a reviewer — and it would break it
9//! silently, four slicers deep, in whichever one was written last.
10//!
11//! So [`Slicer::condense`] is a **provided** method. An implementation writes
12//! [`slice`](Slicer::slice) and never sees the cap enforcement, the CAS write,
13//! or the condensate assembly; `condense` does all three around it. Adding a
14//! fifth slicer therefore cannot forget any of them, which is the only way a
15//! guarantee spread across implementations stays true.
16//!
17//! # Over the cap is an error, never a truncation
18//!
19//! [`SliceError::OverCap`] is deliberately not recoverable by chopping the tail
20//! off. A slicer that produced too much has misjudged its own output, and
21//! silently cutting the excess would be exactly the naive truncation the whole
22//! layer exists to avoid — the last thing dropped would be the end of the file,
23//! which is where the interesting part usually is. Failing tells the author
24//! their slicer is wrong; truncating tells the model a lie.
25//!
26//! # Counting tokens without a tokeniser
27//!
28//! Caps are in tokens; nothing here can tokenise. [`TokenCount`] is the seam.
29//! The default, [`Estimate`], is deliberately pessimistic — it assumes text
30//! costs more tokens per byte than it usually does — because the failure modes
31//! are asymmetric: over-estimating wastes a little window, under-estimating
32//! breaks Law II at admission time, after the money is spent. A caller with a
33//! real tokeniser injects it and gets exact behaviour.
34//!
35//! # Determinism
36//!
37//! Slicing is a pure function of `(raw, cap)`. No clock, no randomness, no
38//! iteration over a hashed collection. Replay (US-1006) re-slices recorded raw
39//! output and compares digests, so a slicer that varied between runs would make
40//! a replayed run fail to match for reasons that have nothing to do with the
41//! model.
42
43use std::fmt;
44use std::time::Duration;
45
46use async_trait::async_trait;
47use boundgraph_core::{Condensate, CondensatePayload, Digest, SchemaId, SlicerId};
48use serde::{Deserialize, Serialize};
49
50/// Where raw output is kept so that nothing is lost by slicing.
51///
52/// # Why this is declared here
53///
54/// `ARCHITECTURE.md` §5 lists `Store` among the nine extension points, and
55/// feature 10 will own the durable implementation — the journal references
56/// payloads by CAS digest (US-1005) and so does a large condensate payload
57/// (US-0104). None of that exists yet, and US-0701 needs exactly one thing from
58/// it: somewhere to put the original before the copy is cut down.
59///
60/// So the boundary is declared here, as narrowly as it can be, exactly as the
61/// runtime declares [`LedgerSink`](boundgraph_runtime::LedgerSink) for the log it does
62/// not own. Feature 10's store will satisfy it unchanged.
63#[async_trait]
64pub trait Store: Send + Sync + fmt::Debug {
65    /// Write bytes, returning their content address.
66    ///
67    /// Idempotent by construction: the address *is* the content, so storing the
68    /// same bytes twice is storing them once.
69    ///
70    /// # Errors
71    ///
72    /// [`StoreError`] if the bytes could not be written. The caller must treat
73    /// that as the slice not having happened — a condensate whose `raw` points
74    /// at nothing is worse than no condensate, because it claims an original
75    /// that cannot be produced.
76    async fn put(&self, bytes: &[u8]) -> Result<Digest, StoreError>;
77
78    /// Read bytes back by address.
79    ///
80    /// # Errors
81    ///
82    /// [`StoreError`] if the bytes are not there or could not be read.
83    async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError>;
84}
85
86/// The content-addressed store could not be used.
87#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
88pub enum StoreError {
89    /// Nothing is stored under that address.
90    #[error("nothing is stored at {0}")]
91    NotFound(Digest),
92    /// The store itself failed.
93    #[error("the content store failed: {0}")]
94    Unavailable(String),
95}
96
97/// Turns text into a token count.
98///
99/// See the module documentation for why the default is pessimistic.
100pub trait TokenCount: Send + Sync + fmt::Debug {
101    /// How many tokens `text` is worth.
102    fn count(&self, text: &str) -> usize;
103}
104
105/// The default token estimate: pessimistic, and deterministic.
106///
107/// # How it counts
108///
109/// Three bytes per token, rounded up, plus one token per line. Real byte-level
110/// BPE averages closer to four bytes per token on prose and rather worse on
111/// dense punctuation, which is most of what a slicer handles — code, diffs, and
112/// JSON. The per-line term is there because newlines and the indentation after
113/// them tokenise poorly and a slicer's output is unusually line-dense.
114///
115/// It is an estimate and says so. What matters is the direction of its error:
116/// over-estimating costs a little unused window, under-estimating overruns
117/// `s_max` and breaks Law II after the call has been paid for.
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
119pub struct Estimate;
120
121impl TokenCount for Estimate {
122    fn count(&self, text: &str) -> usize {
123        if text.is_empty() {
124            return 0;
125        }
126        let bytes = text.len();
127        let lines = text.lines().count();
128        bytes.div_ceil(3) + lines
129    }
130}
131
132/// Counts a token per byte.
133///
134/// The only counter that is a genuine upper bound: byte-level BPE never emits
135/// more tokens than there are bytes, because merging is all it does. Far too
136/// conservative for ordinary use — it would spend a 400-token cap on 400 bytes
137/// — but it is the right answer when a cap must not be exceeded under any
138/// tokeniser at all.
139#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
140pub struct ByteCeiling;
141
142impl TokenCount for ByteCeiling {
143    fn count(&self, text: &str) -> usize {
144        text.len()
145    }
146}
147
148/// What running something produced, beyond its output.
149///
150/// Carried separately from the bytes because US-0703 requires exit status and
151/// timing to survive *regardless of the cap*: they are a handful of tokens and
152/// they are the first thing anyone asks about a failed command, so a slicer
153/// that dropped them to fit more log lines would have optimised the wrong way.
154#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
155pub struct CommandOutcome {
156    /// The command line, as invoked.
157    pub command: String,
158    /// The process exit status. `None` where it was killed by a signal or never
159    /// finished.
160    pub exit: Option<i32>,
161    /// How long it ran.
162    pub duration: Duration,
163}
164
165impl CommandOutcome {
166    /// Record a command that finished with a status.
167    pub fn new(command: impl Into<String>, exit: i32, duration: Duration) -> Self {
168        Self {
169            command: command.into(),
170            exit: Some(exit),
171            duration,
172        }
173    }
174
175    /// Whether it succeeded.
176    pub fn succeeded(&self) -> bool {
177        self.exit == Some(0)
178    }
179
180    /// A one-line summary, always retained.
181    pub fn summary(&self) -> String {
182        let status = match self.exit {
183            Some(0) => "ok".to_string(),
184            Some(code) => format!("exit {code}"),
185            None => "killed".to_string(),
186        };
187        format!(
188            "$ {} — {status} in {}ms",
189            self.command,
190            self.duration.as_millis()
191        )
192    }
193}
194
195/// Raw tool output, before slicing.
196#[derive(Debug, Clone, PartialEq, Eq)]
197pub struct Raw {
198    /// The complete, unmodified bytes.
199    pub bytes: Vec<u8>,
200    /// What produced them, where it was a command.
201    pub command: Option<CommandOutcome>,
202    /// A hint at the content, e.g. a file name or language, for slicers that
203    /// can use one. Never load-bearing: a slicer is chosen by the flow.
204    pub hint: Option<String>,
205}
206
207impl Raw {
208    /// Raw bytes with no provenance.
209    pub fn new(bytes: impl Into<Vec<u8>>) -> Self {
210        Self {
211            bytes: bytes.into(),
212            command: None,
213            hint: None,
214        }
215    }
216
217    /// Raw output from a command.
218    #[must_use]
219    pub fn from_command(mut self, outcome: CommandOutcome) -> Self {
220        self.command = Some(outcome);
221        self
222    }
223
224    /// Hint at what this is.
225    #[must_use]
226    pub fn hinting(mut self, hint: impl Into<String>) -> Self {
227        self.hint = Some(hint.into());
228        self
229    }
230
231    /// The bytes as text, replacing anything that is not valid UTF-8.
232    ///
233    /// Lossy rather than failing: tool output is frequently *nearly* UTF-8 —
234    /// a stray byte in a filename, a terminal escape — and refusing to slice a
235    /// build log over one bad byte would be worse than showing a replacement
236    /// character. The original bytes are in the CAS either way.
237    pub fn text(&self) -> std::borrow::Cow<'_, str> {
238        String::from_utf8_lossy(&self.bytes)
239    }
240
241    /// How large the original was.
242    pub fn len(&self) -> usize {
243        self.bytes.len()
244    }
245
246    /// Whether there was any output at all.
247    pub fn is_empty(&self) -> bool {
248        self.bytes.is_empty()
249    }
250}
251
252/// What a slicer produced, before it becomes a [`Condensate`].
253///
254/// The slicer says what it made and whether it dropped anything; `condense`
255/// turns that into the addressed, typed value the rest of the system sees.
256#[derive(Debug, Clone, PartialEq)]
257pub struct Slice {
258    /// The shape of the summary.
259    pub schema: SchemaId,
260    /// The summary itself.
261    pub payload: serde_json::Value,
262    /// Whether information was dropped.
263    ///
264    /// Set honestly. A slicer that under-reports this is telling a downstream
265    /// step it has the whole picture when it does not, and nothing downstream
266    /// can detect the lie — which is why every slicer in this crate has a test
267    /// that a lossless slice reports `false` and a dropping one reports `true`.
268    pub lossy: bool,
269}
270
271impl Slice {
272    /// A slice that dropped nothing.
273    pub fn complete(schema: SchemaId, payload: serde_json::Value) -> Self {
274        Self {
275            schema,
276            payload,
277            lossy: false,
278        }
279    }
280
281    /// A slice that dropped something.
282    pub fn lossy(schema: SchemaId, payload: serde_json::Value) -> Self {
283        Self {
284            schema,
285            payload,
286            lossy: true,
287        }
288    }
289
290    /// The rendered form, which is what the cap is measured against.
291    ///
292    /// A condensate's payload is structured, but what reaches a model is its
293    /// serialisation — so that is what has to fit. Measuring the structure
294    /// instead would let a slicer pass the check and overrun the window.
295    pub fn rendered(&self) -> String {
296        serde_json::to_string(&self.payload).unwrap_or_default()
297    }
298}
299
300/// Slicing failed.
301#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
302#[non_exhaustive]
303pub enum SliceError {
304    /// The slicer produced more than the cap allows.
305    ///
306    /// A bug in the slicer, not a condition to handle: see the module
307    /// documentation for why this is not silently truncated.
308    #[error("slicer {slicer} produced {produced} tokens against a cap of {cap}")]
309    OverCap {
310        /// Which slicer overran.
311        slicer: String,
312        /// What it produced.
313        produced: usize,
314        /// What it was allowed.
315        cap: usize,
316    },
317
318    /// The cap is too small to say anything at all.
319    ///
320    /// Distinct from [`OverCap`](Self::OverCap): the slicer did nothing wrong,
321    /// the step declared a window that cannot hold even a summary. That is a
322    /// flow authoring error and the message says so.
323    #[error("a cap of {cap} tokens cannot hold even a summary (needs at least {minimum})")]
324    CapTooSmall {
325        /// What was allowed.
326        cap: usize,
327        /// The least that would work.
328        minimum: usize,
329    },
330
331    /// The raw output could not be preserved.
332    #[error("the raw output could not be stored: {0}")]
333    Store(#[from] StoreError),
334
335    /// The input was not what this slicer handles.
336    ///
337    /// Carries the reason so a caller can decide to fall back — which is what
338    /// the AST slicer does when a file does not parse (US-0702).
339    #[error("{slicer} cannot slice this input: {detail}")]
340    Unsuitable {
341        /// Which slicer declined.
342        slicer: String,
343        /// Why.
344        detail: String,
345    },
346}
347
348impl SliceError {
349    /// Whether another slicer might succeed where this one did not.
350    ///
351    /// True only for [`Unsuitable`](Self::Unsuitable). An over-cap slicer is
352    /// broken and a too-small cap is a flow error; neither is fixed by trying
353    /// something else, and falling back on them would hide a real problem.
354    pub fn can_fall_back(&self) -> bool {
355        matches!(self, Self::Unsuitable { .. })
356    }
357}
358
359/// Layer 4: turns raw output into a size-capped, typed summary.
360///
361/// Implement [`slice`](Self::slice) and [`id`](Self::id). Do not implement
362/// [`condense`](Self::condense) — it is what enforces the contract, and the
363/// whole design of this module is that an implementation cannot opt out of it.
364#[async_trait]
365pub trait Slicer: Send + Sync + fmt::Debug {
366    /// Which slicer this is, and at what version.
367    ///
368    /// Recorded into every condensate, so re-slicing the same raw output later
369    /// can be checked against the version that originally produced it.
370    fn id(&self) -> SlicerId;
371
372    /// How this slicer counts tokens.
373    ///
374    /// Defaults to [`Estimate`]. Override to inject a real tokeniser.
375    fn counter(&self) -> &dyn TokenCount {
376        &Estimate
377    }
378
379    /// The least a summary from this slicer can cost.
380    ///
381    /// Used to turn an impossible cap into [`SliceError::CapTooSmall`] before
382    /// any work happens, so the author is told their step's window is too small
383    /// rather than handed a mysterious over-cap failure.
384    fn minimum_cap(&self) -> usize {
385        16
386    }
387
388    /// Produce the summary. The part an implementation writes.
389    ///
390    /// Must be a pure function of `(raw, cap)` — see the module documentation
391    /// on determinism.
392    ///
393    /// # Errors
394    ///
395    /// [`SliceError::Unsuitable`] if this slicer does not handle the input.
396    async fn slice(&self, raw: &Raw, cap: usize) -> Result<Slice, SliceError>;
397
398    /// Preserve the original, slice it, and check the result fits.
399    ///
400    /// In that order, and the order matters: the raw bytes reach the store
401    /// *before* anything is dropped, so a slicer that fails part-way through
402    /// still leaves the original recoverable.
403    ///
404    /// # Errors
405    ///
406    /// [`SliceError::Store`] if the original could not be preserved,
407    /// [`SliceError::CapTooSmall`] if the cap cannot hold a summary,
408    /// [`SliceError::OverCap`] if the slicer overran it, or whatever
409    /// [`slice`](Self::slice) returned.
410    async fn condense(
411        &self,
412        raw: &Raw,
413        cap: usize,
414        store: &dyn Store,
415    ) -> Result<Condensate, SliceError> {
416        let minimum = self.minimum_cap();
417        if cap < minimum {
418            return Err(SliceError::CapTooSmall { cap, minimum });
419        }
420
421        // Before slicing, so nothing is lost even if slicing then fails.
422        let digest = store.put(&raw.bytes).await?;
423
424        let slice = self.slice(raw, cap).await?;
425
426        let produced = self.counter().count(&slice.rendered());
427        if produced > cap {
428            return Err(SliceError::OverCap {
429                slicer: self.id().name.clone(),
430                produced,
431                cap,
432            });
433        }
434
435        Ok(Condensate::new(
436            slice.schema,
437            CondensatePayload::inline(&slice.payload).map_err(|e| SliceError::Unsuitable {
438                slicer: self.id().name.clone(),
439                detail: format!("the summary could not be encoded: {e}"),
440            })?,
441            digest,
442            cap,
443            slice.lossy,
444            self.id(),
445        ))
446    }
447}
448
449/// A store that keeps everything in memory.
450///
451/// What the M3 milestone runs on, and a first-class part of the crate for the
452/// same reason the mock reasoner is: a test, a cassette replay, and a policy
453/// check all want a store that needs no filesystem.
454#[derive(Debug, Default)]
455pub struct MemoryStore {
456    blobs: std::sync::Mutex<std::collections::BTreeMap<Digest, Vec<u8>>>,
457}
458
459impl MemoryStore {
460    /// An empty store.
461    pub fn new() -> Self {
462        Self::default()
463    }
464
465    /// How many distinct blobs are held.
466    ///
467    /// Distinct, not written: content addressing means storing the same bytes
468    /// twice stores them once, and this is how that is asserted.
469    pub fn len(&self) -> usize {
470        self.blobs.lock().expect("store lock").len()
471    }
472
473    /// Whether nothing has been stored.
474    pub fn is_empty(&self) -> bool {
475        self.len() == 0
476    }
477}
478
479#[async_trait]
480impl Store for MemoryStore {
481    async fn put(&self, bytes: &[u8]) -> Result<Digest, StoreError> {
482        let digest = Digest::of(bytes);
483        self.blobs
484            .lock()
485            .expect("store lock")
486            .insert(digest, bytes.to_vec());
487        Ok(digest)
488    }
489
490    async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError> {
491        self.blobs
492            .lock()
493            .expect("store lock")
494            .get(digest)
495            .cloned()
496            .ok_or(StoreError::NotFound(*digest))
497    }
498}
499
500#[cfg(test)]
501mod tests {
502    use super::*;
503    use serde_json::json;
504
505    /// A slicer that emits exactly what it is told to, including too much.
506    #[derive(Debug)]
507    struct Fixed {
508        payload: serde_json::Value,
509        lossy: bool,
510    }
511
512    impl Fixed {
513        fn new(payload: serde_json::Value) -> Self {
514            Self {
515                payload,
516                lossy: false,
517            }
518        }
519    }
520
521    #[async_trait]
522    impl Slicer for Fixed {
523        fn id(&self) -> SlicerId {
524            SlicerId::new("fixed", 1)
525        }
526
527        async fn slice(&self, _raw: &Raw, _cap: usize) -> Result<Slice, SliceError> {
528            Ok(Slice {
529                schema: SchemaId::new("test.fixed.v1", 1),
530                payload: self.payload.clone(),
531                lossy: self.lossy,
532            })
533        }
534    }
535
536    fn raw() -> Raw {
537        Raw::new(b"the complete original output".to_vec())
538    }
539
540    // ── the contract ────────────────────────────────────────────────────────
541
542    #[tokio::test]
543    async fn condense_records_the_slicer_id_and_the_cap() {
544        let store = MemoryStore::new();
545        let condensate = Fixed::new(json!("small"))
546            .condense(&raw(), 100, &store)
547            .await
548            .unwrap();
549
550        assert_eq!(condensate.slicer, SlicerId::new("fixed", 1));
551        assert_eq!(condensate.schema, SchemaId::new("test.fixed.v1", 1));
552        assert_eq!(condensate.cap, 100);
553    }
554
555    /// The criterion: raw reaches the CAS, so nothing is lost by slicing.
556    #[tokio::test]
557    async fn the_original_is_preserved_and_addressed_by_the_condensate() {
558        let store = MemoryStore::new();
559        let raw = raw();
560        let condensate = Fixed::new(json!("small"))
561            .condense(&raw, 100, &store)
562            .await
563            .unwrap();
564
565        assert_eq!(condensate.raw, Digest::of(&raw.bytes));
566        assert_eq!(store.get(&condensate.raw).await.unwrap(), raw.bytes);
567    }
568
569    /// And it gets there *before* slicing, so a slicer that fails still leaves
570    /// the original recoverable.
571    #[tokio::test]
572    async fn the_original_survives_a_slicer_that_fails() {
573        #[derive(Debug)]
574        struct Fails;
575
576        #[async_trait]
577        impl Slicer for Fails {
578            fn id(&self) -> SlicerId {
579                SlicerId::new("fails", 1)
580            }
581            async fn slice(&self, _raw: &Raw, _cap: usize) -> Result<Slice, SliceError> {
582                Err(SliceError::Unsuitable {
583                    slicer: "fails".into(),
584                    detail: "not my kind of input".into(),
585                })
586            }
587        }
588
589        let store = MemoryStore::new();
590        let raw = raw();
591        assert!(Fails.condense(&raw, 100, &store).await.is_err());
592
593        assert_eq!(
594            store.get(&Digest::of(&raw.bytes)).await.unwrap(),
595            raw.bytes,
596            "the original must outlive a failed slice"
597        );
598    }
599
600    /// The criterion the whole module is built around: over the cap is a typed
601    /// error, and it is enforced by the trait rather than by the slicer.
602    #[tokio::test]
603    async fn a_slicer_that_overruns_its_cap_is_refused_not_truncated() {
604        let store = MemoryStore::new();
605        let long = json!("x".repeat(4_000));
606
607        let err = Fixed::new(long)
608            .condense(&raw(), 50, &store)
609            .await
610            .unwrap_err();
611
612        let SliceError::OverCap {
613            slicer,
614            produced,
615            cap,
616        } = err
617        else {
618            panic!("expected OverCap, got {err:?}");
619        };
620        assert_eq!(slicer, "fixed");
621        assert_eq!(cap, 50);
622        assert!(produced > 50);
623        assert!(!err_can_fall_back(produced, cap));
624    }
625
626    fn err_can_fall_back(produced: usize, cap: usize) -> bool {
627        SliceError::OverCap {
628            slicer: "fixed".into(),
629            produced,
630            cap,
631        }
632        .can_fall_back()
633    }
634
635    #[tokio::test]
636    async fn a_cap_too_small_for_any_summary_is_its_own_error() {
637        let store = MemoryStore::new();
638        let err = Fixed::new(json!("small"))
639            .condense(&raw(), 3, &store)
640            .await
641            .unwrap_err();
642
643        assert!(matches!(err, SliceError::CapTooSmall { cap: 3, .. }));
644        assert!(
645            !err.can_fall_back(),
646            "another slicer has the same window problem"
647        );
648    }
649
650    #[tokio::test]
651    async fn only_an_unsuitable_input_invites_a_fallback() {
652        let unsuitable = SliceError::Unsuitable {
653            slicer: "ast".into(),
654            detail: "does not parse".into(),
655        };
656        assert!(unsuitable.can_fall_back());
657    }
658
659    /// Exactly at the cap is allowed; the check is `>`, not `>=`.
660    #[tokio::test]
661    async fn a_slice_exactly_at_the_cap_is_accepted() {
662        let store = MemoryStore::new();
663        let payload = json!("abc");
664        let exact = Estimate.count(&serde_json::to_string(&payload).unwrap());
665
666        assert!(
667            Fixed::new(payload)
668                .condense(&raw(), exact.max(16), &store)
669                .await
670                .is_ok()
671        );
672    }
673
674    #[tokio::test]
675    async fn lossy_is_carried_through_honestly() {
676        let store = MemoryStore::new();
677
678        let clean = Fixed::new(json!("all of it"))
679            .condense(&raw(), 100, &store)
680            .await
681            .unwrap();
682        assert!(!clean.lossy);
683
684        let dropped = Fixed {
685            payload: json!("some of it"),
686            lossy: true,
687        }
688        .condense(&raw(), 100, &store)
689        .await
690        .unwrap();
691        assert!(dropped.lossy);
692        assert!(dropped.needs_raw_for_completeness());
693    }
694
695    /// Replay re-slices recorded output and compares digests, so a slicer that
696    /// varied between runs would break a replay for no reason.
697    #[tokio::test]
698    async fn slicing_the_same_input_twice_gives_the_same_condensate() {
699        let store = MemoryStore::new();
700        let slicer = Fixed::new(json!({ "b": 2, "a": 1 }));
701
702        let first = slicer.condense(&raw(), 100, &store).await.unwrap();
703        for _ in 0..8 {
704            assert_eq!(slicer.condense(&raw(), 100, &store).await.unwrap(), first);
705        }
706    }
707
708    // ── counting ────────────────────────────────────────────────────────────
709
710    #[test]
711    fn the_default_estimate_is_pessimistic_rather_than_accurate() {
712        // Prose really costs about a quarter of its bytes; the estimate says a
713        // third plus a line, and being wrong in that direction is the point.
714        let prose = "the quick brown fox jumps over the lazy dog";
715        assert!(Estimate.count(prose) > prose.len() / 4);
716        assert_eq!(Estimate.count(""), 0);
717    }
718
719    #[test]
720    fn the_estimate_charges_for_lines_because_slicer_output_is_line_dense() {
721        let one_line = "aaaaaaaaaaa";
722        let three_lines = "aaa\naaa\naaa";
723        assert_eq!(
724            one_line.len(),
725            three_lines.len(),
726            "same bytes, different shape"
727        );
728        assert!(Estimate.count(three_lines) > Estimate.count(one_line));
729    }
730
731    /// Byte-level BPE only ever merges, so it cannot emit more tokens than
732    /// there are bytes. That makes this the one counter that is a bound rather
733    /// than an estimate — and, for the same reason, far too conservative to
734    /// use by default.
735    #[test]
736    fn the_byte_ceiling_is_a_genuine_upper_bound() {
737        for text in ["", "a", "hello world", "{\"k\":[1,2,3]}"] {
738            assert_eq!(ByteCeiling.count(text), text.len());
739            assert!(
740                ByteCeiling.count(text) >= Estimate.count(text) || text.len() < 4,
741                "the bound should not sit below the estimate for real inputs"
742            );
743        }
744    }
745
746    // ── the store ───────────────────────────────────────────────────────────
747
748    #[tokio::test]
749    async fn storing_the_same_bytes_twice_stores_them_once() {
750        let store = MemoryStore::new();
751        let a = store.put(b"identical").await.unwrap();
752        let b = store.put(b"identical").await.unwrap();
753
754        assert_eq!(a, b);
755        assert_eq!(store.len(), 1);
756    }
757
758    #[tokio::test]
759    async fn an_address_nothing_was_stored_at_is_a_typed_miss() {
760        let store = MemoryStore::new();
761        let err = store.get(&Digest::of(b"never stored")).await.unwrap_err();
762        assert!(matches!(err, StoreError::NotFound(_)));
763    }
764
765    /// A condensate whose raw points at nothing claims an original that cannot
766    /// be produced, which is worse than no condensate.
767    #[tokio::test]
768    async fn a_store_failure_stops_the_slice() {
769        #[derive(Debug)]
770        struct Broken;
771
772        #[async_trait]
773        impl Store for Broken {
774            async fn put(&self, _bytes: &[u8]) -> Result<Digest, StoreError> {
775                Err(StoreError::Unavailable("disk full".into()))
776            }
777            async fn get(&self, digest: &Digest) -> Result<Vec<u8>, StoreError> {
778                Err(StoreError::NotFound(*digest))
779            }
780        }
781
782        let err = Fixed::new(json!("small"))
783            .condense(&raw(), 100, &Broken)
784            .await
785            .unwrap_err();
786        assert!(matches!(err, SliceError::Store(_)));
787    }
788
789    // ── raw ─────────────────────────────────────────────────────────────────
790
791    #[test]
792    fn raw_text_survives_bytes_that_are_not_utf8() {
793        let raw = Raw::new(vec![b'o', b'k', 0xff, b'!']);
794        assert!(raw.text().contains("ok"));
795        assert_eq!(raw.len(), 4);
796        assert!(!raw.is_empty());
797    }
798
799    #[test]
800    fn a_command_outcome_summarises_status_and_timing() {
801        let ok = CommandOutcome::new("cargo build", 0, Duration::from_millis(1_200));
802        assert!(ok.succeeded());
803        assert!(ok.summary().contains("cargo build"));
804        assert!(ok.summary().contains("ok"));
805        assert!(ok.summary().contains("1200ms"));
806
807        let failed = CommandOutcome::new("cargo test", 101, Duration::from_millis(50));
808        assert!(!failed.succeeded());
809        assert!(failed.summary().contains("exit 101"));
810
811        let killed = CommandOutcome {
812            command: "sleep 100".into(),
813            exit: None,
814            duration: Duration::from_secs(9),
815        };
816        assert!(killed.summary().contains("killed"));
817    }
818}