cratefield_core/
cost.rs

1//! What a classifier call costs (issue #457): who answered
2//! ([`AdapterId`]), under which question ids ([`QuestionKind`]), how many
3//! tokens it spent ([`Tokens`]), what those tokens cost ([`PriceSheet`],
4//! [`Cost`]), and the ledger that totals it ([`CostLedger`]).
5//!
6//! **Tokens are estimated.** The [`Classifier`](crate::Classifier) port
7//! reports no usage — an [`Answer`] carries a value and probabilities, not
8//! a bill — so every [`CallRecord`] carries [`Tokens`] tagged
9//! [`TokenSource::Estimated`]: about four chars per token over what was
10//! sent and what came back (see [`Tokens::estimate`]). The estimate is
11//! good enough to compare two adapters asked the same questions, and it
12//! is not an invoice.
13//!
14//! **Money is an integer.** [`Price`] is pico-USD (10^-12 USD) per token,
15//! so a vendor's `$X.YZ per million tokens` written in micro-USD is the
16//! same integer ([`Price::per_million_tokens`]). [`Cost`] saturates rather
17//! than wraps, and [`LedgerTotals::pico_usd`] is a `u128` so a lifetime
18//! total cannot saturate where one call would not.
19//!
20//! **An adapter with no price is unknown, not free.**
21//! [`PriceSheet::cost_of`] answers `None` for it, and totals count those
22//! calls in [`LedgerTotals::unpriced_calls`] instead of summing a zero:
23//! a cheap-vs-expensive argument built on a silent zero is the mistake
24//! this module exists to prevent.
25
26use std::collections::BTreeMap;
27use std::fmt;
28use std::sync::Arc;
29
30use serde::{Deserialize, Serialize};
31
32use crate::ports::{Answer, AnswerValue, ClassifierProfile, Question};
33
34/// Pico-USD in one USD.
35const PICO_PER_USD: u64 = 1_000_000_000_000;
36
37/// Chars per token the estimate assumes — the same rule of thumb
38/// [`DEFAULT_MAX_STATE_CHARS`](crate::DEFAULT_MAX_STATE_CHARS) is sized by.
39const CHARS_PER_TOKEN: usize = 4;
40
41/// The name a venture wires a classifier adapter under — the key prices,
42/// thresholds and agreement evidence are kept by. The port does not name
43/// its adapters, so the wiring does.
44#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
45pub struct AdapterId(String);
46
47impl AdapterId {
48    /// An id from its name.
49    #[must_use]
50    pub fn new(id: impl Into<String>) -> Self {
51        Self(id.into())
52    }
53
54    /// The id as a string.
55    #[must_use]
56    pub fn as_str(&self) -> &str {
57        &self.0
58    }
59}
60
61impl fmt::Display for AdapterId {
62    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
63        f.write_str(&self.0)
64    }
65}
66
67/// The id a question is asked under — the key of the map
68/// [`Classifier::ask`](crate::Classifier::ask) takes. A venture asks the
69/// same kind of question under the same id, so agreement is measured and
70/// routing is decided per id: a classifier reliable on one kind is not
71/// therefore reliable on the next.
72#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
73pub struct QuestionKind(String);
74
75impl QuestionKind {
76    /// A kind from the question id.
77    #[must_use]
78    pub fn new(kind: impl Into<String>) -> Self {
79        Self(kind.into())
80    }
81
82    /// The kind as a string.
83    #[must_use]
84    pub fn as_str(&self) -> &str {
85        &self.0
86    }
87}
88
89impl fmt::Display for QuestionKind {
90    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
91        f.write_str(&self.0)
92    }
93}
94
95/// Where a token count came from.
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
97#[non_exhaustive]
98pub enum TokenSource {
99    /// Counted from the text by [`Tokens::estimate`], because the port
100    /// reports no usage. The only source today; a provider-reported count
101    /// would be a second variant.
102    Estimated,
103}
104
105/// The tokens one call spent, and where the count came from.
106#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
107pub struct Tokens {
108    pub input: u64,
109    pub output: u64,
110    pub source: TokenSource,
111}
112
113impl Tokens {
114    /// Estimates one call at `ceil(chars / 4)` per side.
115    ///
116    /// Input is the `state` **as the adapter sends it** — truncated by
117    /// `profile` the way the adapter truncates it, so an over-long state
118    /// is not billed for text that never left — plus every question's id,
119    /// instructions and criteria or levels (names and meanings). Output is
120    /// the answers rendered as id, value and every probability label with
121    /// its number to two places; `None` (a failed call) is zero output.
122    ///
123    /// An estimate, tagged [`TokenSource::Estimated`]: each adapter wraps
124    /// the text in its own prompt and tokenizes it its own way. It compares
125    /// adapters asked the same questions fairly, and it is not what the
126    /// vendor will invoice.
127    #[must_use]
128    pub fn estimate(
129        profile: &ClassifierProfile,
130        state: &str,
131        questions: &BTreeMap<String, Question>,
132        answers: Option<&BTreeMap<String, Answer>>,
133    ) -> Self {
134        let (sent, _) = profile.truncate(state);
135        let mut input = sent.chars().count();
136        for (id, question) in questions {
137            input += id.chars().count() + question.instructions().chars().count();
138            let options: Vec<(&String, &String)> = match question {
139                Question::Choice { criteria, .. } => criteria.iter().collect(),
140                Question::Score { levels, .. } => levels.iter().map(|(n, m)| (n, m)).collect(),
141                Question::Noul { .. } => Vec::new(),
142            };
143            for (name, meaning) in options {
144                input += name.chars().count() + meaning.chars().count();
145            }
146        }
147        let output = answers.map_or(0, |answers| {
148            answers
149                .iter()
150                .map(|(id, answer)| {
151                    let probabilities: usize = answer
152                        .probabilities
153                        .iter()
154                        .map(|(label, p)| label.chars().count() + format!("{p:.2}").len())
155                        .sum();
156                    id.chars().count() + value_text(&answer.value).chars().count() + probabilities
157                })
158                .sum()
159        });
160        Self {
161            input: tokens_for(input),
162            output: tokens_for(output),
163            source: TokenSource::Estimated,
164        }
165    }
166}
167
168/// `ceil(chars / 4)`, saturating into a `u64`.
169fn tokens_for(chars: usize) -> u64 {
170    u64::try_from(chars.div_ceil(CHARS_PER_TOKEN)).unwrap_or(u64::MAX)
171}
172
173/// An answer's value as text: the label, the score, or `true`/`false`.
174pub(crate) fn value_text(value: &AnswerValue) -> String {
175    match value {
176        AnswerValue::Choice(label) => label.clone(),
177        AnswerValue::Score(score) => score.to_string(),
178        AnswerValue::Noul(verdict) => verdict.to_string(),
179    }
180}
181
182/// What one token costs on one adapter, in pico-USD, per side. Pricing is
183/// config, pasted off the vendor's page through
184/// [`Price::per_million_tokens`].
185#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
186pub struct Price {
187    pub input_pico_usd_per_token: u64,
188    pub output_pico_usd_per_token: u64,
189}
190
191impl Price {
192    /// A **known** zero, such as a self-hosted model — not the same thing
193    /// as an adapter missing from the sheet.
194    pub const FREE: Price = Price::per_million_tokens(0, 0);
195
196    /// A vendor's published prices: `$3.00 per million input tokens` is
197    /// `per_million_tokens(3_000_000, ..)`, because micro-USD per million
198    /// tokens is numerically pico-USD per token.
199    #[must_use]
200    pub const fn per_million_tokens(input_micro_usd: u64, output_micro_usd: u64) -> Price {
201        Price {
202            input_pico_usd_per_token: input_micro_usd,
203            output_pico_usd_per_token: output_micro_usd,
204        }
205    }
206
207    /// The cost of `tokens`, saturating rather than wrapping.
208    #[must_use]
209    pub const fn cost(&self, tokens: &Tokens) -> Cost {
210        Cost(
211            tokens
212                .input
213                .saturating_mul(self.input_pico_usd_per_token)
214                .saturating_add(tokens.output.saturating_mul(self.output_pico_usd_per_token)),
215        )
216    }
217}
218
219/// The pico-USD cost of one call: a `u64`, saturating near $18.4 million,
220/// which no single classification spends.
221#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
222pub struct Cost(u64);
223
224impl Cost {
225    /// What a known-free adapter spends.
226    pub const ZERO: Cost = Cost(0);
227
228    /// Wraps a raw pico-USD amount.
229    #[must_use]
230    pub const fn from_pico_usd(pico_usd: u64) -> Cost {
231        Cost(pico_usd)
232    }
233
234    /// The raw amount in pico-USD.
235    #[must_use]
236    pub const fn pico_usd(&self) -> u64 {
237        self.0
238    }
239}
240
241impl fmt::Display for Cost {
242    /// All twelve decimals: fewer would round a small call to `$0.00`.
243    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
244        write!(
245            f,
246            "${}.{:012}",
247            self.0 / PICO_PER_USD,
248            self.0 % PICO_PER_USD
249        )
250    }
251}
252
253/// What each adapter costs, wired once where the adapters are wired.
254#[derive(Debug, Clone, Default, PartialEq)]
255pub struct PriceSheet(BTreeMap<AdapterId, Price>);
256
257impl PriceSheet {
258    /// An empty sheet: every adapter unpriced.
259    #[must_use]
260    pub fn new() -> Self {
261        Self::default()
262    }
263
264    /// Prices `adapter`, replacing any price it had.
265    #[must_use]
266    pub fn with(mut self, adapter: AdapterId, price: Price) -> Self {
267        self.0.insert(adapter, price);
268        self
269    }
270
271    /// What `tokens` cost on `adapter` — `None` when the adapter has no
272    /// price: unknown, never free.
273    #[must_use]
274    pub fn cost_of(&self, adapter: &AdapterId, tokens: &Tokens) -> Option<Cost> {
275        self.0.get(adapter).map(|price| price.cost(tokens))
276    }
277}
278
279/// Which side of a routing decision a recorded call was on.
280#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
281pub enum CallRole {
282    /// At least one of its answers was accepted to serve, or its error
283    /// reached the caller.
284    Served,
285    /// Asked only to compare: the cheap half of shadow mode, or either
286    /// half of a [`measure_agreement`](crate::measure_agreement) run.
287    Shadow,
288    /// Asked to serve, and none of its answers was accepted: every one
289    /// escalated, or the call failed. The ledger's
290    /// [`InMemoryLedger::role_totals`] row for this role is what routing
291    /// paid for nothing.
292    ///
293    /// A role is set when the call returns and is recorded then, so a
294    /// caller that drops the future later loses no cost. A cheap call
295    /// recorded as `Served` stays so even if the expensive escalation that
296    /// follows fails and its answers never reach the caller.
297    Discarded,
298}
299
300/// How a recorded call ended. A failed call is recorded too: a cheap
301/// adapter that fails often is not cheap.
302#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
303pub enum CallOutcome {
304    Ok,
305    Failed,
306}
307
308/// One classifier call as the ledger receives it.
309///
310/// `cost` is `None` when the adapter has no price on the sheet, and for a
311/// failed call — whether a vendor bills a failed request is its own
312/// business, so the router does not guess.
313#[derive(Debug, Clone, PartialEq, Eq)]
314#[non_exhaustive]
315pub struct CallRecord {
316    pub adapter: AdapterId,
317    /// Every question id the call carried, in order.
318    pub kinds: Vec<QuestionKind>,
319    pub tokens: Tokens,
320    pub cost: Option<Cost>,
321    pub role: CallRole,
322    pub outcome: CallOutcome,
323}
324
325/// Sinks what classifier calls cost.
326///
327/// Sync and infallible on purpose: recording what a decision cost must not
328/// be able to fail the decision or add a network hop. A durable sink
329/// buffers and flushes on its own schedule.
330pub trait CostLedger: Send + Sync {
331    /// Records one call. Must not block on I/O or fail.
332    fn record(&self, record: CallRecord);
333}
334
335/// The price sheet and the ledger every call is recorded to — what the
336/// router, the shadow classifier and the corpus runner are all given, so
337/// no classifier call through them goes unrecorded.
338#[derive(Clone)]
339pub struct Accounting {
340    prices: PriceSheet,
341    ledger: Arc<dyn CostLedger>,
342}
343
344impl Accounting {
345    /// Prices calls off `prices` and records them to `ledger`.
346    #[must_use]
347    pub fn new(prices: PriceSheet, ledger: Arc<dyn CostLedger>) -> Self {
348        Self { prices, ledger }
349    }
350
351    /// Records one call: estimated tokens, priced when it answered.
352    pub(crate) fn record(
353        &self,
354        adapter: &AdapterId,
355        profile: &ClassifierProfile,
356        state: &str,
357        questions: &BTreeMap<String, Question>,
358        answers: Option<&BTreeMap<String, Answer>>,
359        role: CallRole,
360    ) {
361        let tokens = Tokens::estimate(profile, state, questions, answers);
362        self.ledger.record(CallRecord {
363            adapter: adapter.clone(),
364            kinds: questions.keys().map(QuestionKind::new).collect(),
365            tokens,
366            cost: answers.and_then(|_| self.prices.cost_of(adapter, &tokens)),
367            role,
368            outcome: if answers.is_some() {
369                CallOutcome::Ok
370            } else {
371                CallOutcome::Failed
372            },
373        });
374    }
375}
376
377impl fmt::Debug for Accounting {
378    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
379        f.debug_struct("Accounting")
380            .field("prices", &self.prices)
381            .finish_non_exhaustive()
382    }
383}
384
385/// Totals over a set of records.
386#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
387pub struct LedgerTotals {
388    pub calls: u64,
389    /// Estimated, like every count in [`Tokens`].
390    pub input_tokens: u64,
391    pub output_tokens: u64,
392    /// Sum of the known costs only.
393    pub pico_usd: u128,
394    /// Answered calls the sheet could not price — a wiring gap. Zero is
395    /// the only healthy value.
396    pub unpriced_calls: u64,
397    /// Calls the adapter errored on. Not a wiring gap, so not counted in
398    /// `unpriced_calls`.
399    pub failed_calls: u64,
400}
401
402impl LedgerTotals {
403    fn add(&mut self, record: &CallRecord) {
404        self.calls = self.calls.saturating_add(1);
405        self.input_tokens = self.input_tokens.saturating_add(record.tokens.input);
406        self.output_tokens = self.output_tokens.saturating_add(record.tokens.output);
407        match (record.cost, record.outcome) {
408            (Some(cost), _) => {
409                self.pico_usd = self.pico_usd.saturating_add(u128::from(cost.pico_usd()));
410            }
411            (None, CallOutcome::Ok) => self.unpriced_calls = self.unpriced_calls.saturating_add(1),
412            (None, CallOutcome::Failed) => {}
413        }
414        if record.outcome == CallOutcome::Failed {
415            self.failed_calls = self.failed_calls.saturating_add(1);
416        }
417    }
418}
419
420/// A [`CostLedger`] that keeps every record in memory, oldest first.
421/// Process-local — one isolate's view on Workers — which suits a
422/// measurement window and is not a billing system.
423#[derive(Debug, Default)]
424pub struct InMemoryLedger {
425    #[expect(
426        clippy::disallowed_types,
427        reason = "the ledger is an explicitly wired recording cell, not ambient request state (ADR 0007)"
428    )]
429    records: std::sync::Mutex<Vec<CallRecord>>,
430}
431
432impl InMemoryLedger {
433    /// An empty ledger.
434    #[must_use]
435    pub fn new() -> Self {
436        Self::default()
437    }
438
439    /// Recovers from a poison: throwing away every record because one
440    /// append panicked would be the dishonest choice for a ledger.
441    fn lock(&self) -> std::sync::MutexGuard<'_, Vec<CallRecord>> {
442        self.records
443            .lock()
444            .unwrap_or_else(std::sync::PoisonError::into_inner)
445    }
446
447    /// Every record so far, oldest first.
448    #[must_use]
449    pub fn records(&self) -> Vec<CallRecord> {
450        self.lock().clone()
451    }
452
453    /// Totals per adapter.
454    #[must_use]
455    pub fn totals(&self) -> BTreeMap<AdapterId, LedgerTotals> {
456        let mut totals: BTreeMap<AdapterId, LedgerTotals> = BTreeMap::new();
457        for record in self.lock().iter() {
458            totals
459                .entry(record.adapter.clone())
460                .or_default()
461                .add(record);
462        }
463        totals
464    }
465
466    /// Totals per [`CallRole`]: the `Discarded` row is what escalation
467    /// wasted, the `Shadow` row what measuring cost.
468    #[must_use]
469    pub fn role_totals(&self) -> BTreeMap<CallRole, LedgerTotals> {
470        let mut totals: BTreeMap<CallRole, LedgerTotals> = BTreeMap::new();
471        for record in self.lock().iter() {
472            totals.entry(record.role).or_default().add(record);
473        }
474        totals
475    }
476}
477
478impl CostLedger for InMemoryLedger {
479    fn record(&self, record: CallRecord) {
480        self.lock().push(record);
481    }
482}
483
484#[cfg(test)]
485mod tests {
486    use super::*;
487    use crate::ports::Calibration;
488
489    fn tokens(input: u64, output: u64) -> Tokens {
490        Tokens {
491            input,
492            output,
493            source: TokenSource::Estimated,
494        }
495    }
496
497    #[test]
498    fn a_published_price_maps_one_to_one_and_displays_every_pico_usd() {
499        let price = Price::per_million_tokens(3_000_000, 15_000_000);
500        assert_eq!(
501            price.cost(&tokens(1_000_000, 0)).to_string(),
502            "$3.000000000000"
503        );
504        assert_eq!(
505            price.cost(&tokens(1_000_000, 1_000_000)).pico_usd(),
506            18 * PICO_PER_USD
507        );
508        assert_eq!(Cost::from_pico_usd(1).to_string(), "$0.000000000001");
509        assert_eq!(
510            Price::per_million_tokens(u64::MAX, u64::MAX)
511                .cost(&tokens(u64::MAX, u64::MAX))
512                .pico_usd(),
513            u64::MAX,
514            "saturates, never wraps into a credit"
515        );
516    }
517
518    #[test]
519    fn an_unpriced_adapter_is_unknown_and_a_free_one_is_zero() {
520        let free = AdapterId::new("self-hosted");
521        let sheet = PriceSheet::new().with(free.clone(), Price::FREE);
522        assert_eq!(sheet.cost_of(&free, &tokens(10, 10)), Some(Cost::ZERO));
523        assert_eq!(
524            sheet.cost_of(&AdapterId::new("mystery"), &tokens(10, 10)),
525            None
526        );
527    }
528
529    #[test]
530    fn tokens_are_estimated_from_the_truncated_state_the_questions_and_the_answers() {
531        let questions = BTreeMap::from([(
532            "ok".to_owned(),
533            Question::Choice {
534                instructions: "pick".to_owned(),
535                criteria: BTreeMap::from([
536                    ("a".to_owned(), "yes".to_owned()),
537                    ("b".to_owned(), "no".to_owned()),
538                ]),
539            },
540        )]);
541        let profile = ClassifierProfile::new(Calibration::Classifier, 8);
542        // 8 state chars kept of 20 + "ok" + "pick" + "a"+"yes" + "b"+"no" = 22 chars.
543        let asked = Tokens::estimate(&profile, &"x".repeat(20), &questions, None);
544        assert_eq!(
545            asked,
546            tokens(6, 0),
547            "ceil(22 / 4), and no output for a failed call"
548        );
549
550        let answers = BTreeMap::from([(
551            "ok".to_owned(),
552            Answer::choice(
553                "a",
554                BTreeMap::from([("a".to_owned(), 0.9), ("b".to_owned(), 0.1)]),
555            ),
556        )]);
557        // "ok" + "a" + "a0.90" + "b0.10" = 13 chars.
558        let answered = Tokens::estimate(&profile, "", &questions, Some(&answers));
559        assert_eq!(answered.output, 4);
560        assert_eq!(answered.source, TokenSource::Estimated);
561    }
562
563    fn record(
564        adapter: &str,
565        cost: Option<Cost>,
566        role: CallRole,
567        outcome: CallOutcome,
568    ) -> CallRecord {
569        CallRecord {
570            adapter: AdapterId::new(adapter),
571            kinds: vec![QuestionKind::new("topic")],
572            tokens: tokens(100, 10),
573            cost,
574            role,
575            outcome,
576        }
577    }
578
579    #[test]
580    fn totals_separate_unpriced_from_failed_and_do_not_saturate_at_one_calls_ceiling() {
581        let ledger = InMemoryLedger::new();
582        let huge = Some(Cost::from_pico_usd(u64::MAX));
583        ledger.record(record("cheap", huge, CallRole::Served, CallOutcome::Ok));
584        ledger.record(record("cheap", huge, CallRole::Discarded, CallOutcome::Ok));
585        ledger.record(record("cheap", None, CallRole::Served, CallOutcome::Ok));
586        ledger.record(record("cheap", None, CallRole::Shadow, CallOutcome::Failed));
587
588        let cheap = ledger.totals()[&AdapterId::new("cheap")];
589        assert_eq!(cheap.calls, 4);
590        assert_eq!(cheap.pico_usd, u128::from(u64::MAX) * 2);
591        assert_eq!(
592            cheap.unpriced_calls, 1,
593            "only the answered, unpriced call is a gap"
594        );
595        assert_eq!(cheap.failed_calls, 1);
596        assert_eq!(cheap.input_tokens, 400);
597
598        let roles = ledger.role_totals();
599        assert_eq!(roles[&CallRole::Discarded].calls, 1);
600        assert_eq!(roles[&CallRole::Served].calls, 2);
601        assert_eq!(ledger.records()[3].role, CallRole::Shadow, "oldest first");
602    }
603}
cratefield_core/cost.rs

cratefield_core/
cost.rs
