Skip to main content

yoagent_workers/
ai.rs

1//! The Workers AI binding (`env.AI`) as a decision backend.
2//!
3//! [`AiBackend`] calls `env.AI.run(model, input)` with the SystemOne request
4//! yoagent builds and parses the result with [`parse_systemone_response`],
5//! so answers are validated and spend is recorded as for the REST preset
6//! ([`DecisionModel::clef`]). [`clef`] and [`clef_flash`] are the ready-made
7//! models; they are priced at the rate the price table holds when they are
8//! built.
9
10use async_trait::async_trait;
11use js_sys::{Function, Promise, Reflect, JSON};
12use wasm_bindgen::{JsCast, JsValue};
13use wasm_bindgen_futures::JsFuture;
14use yoagent::decision::{
15    parse_systemone_response, Capabilities, DecisionBackend, DecisionError, DecisionModel,
16    Evaluation, QuestionKind, Request,
17};
18use yoagent::retry::RetryConfig;
19
20/// Clef (27B) in the Workers AI catalog.
21pub const CLEF: &str = "@cf/cloudflare/clef";
22/// Clef Flash (9B, faster) in the Workers AI catalog.
23pub const CLEF_FLASH: &str = "@cf/cloudflare/clef-flash";
24
25/// The `prices.json` provider key for Workers AI's models.
26const PRICE_PROVIDER: &str = "cloudflare";
27
28/// Workers AI's error codes, as documented for its REST API
29/// (<https://developers.cloudflare.com/workers-ai/platform/errors/>). A
30/// binding error is read for a leading `NNNN:` code; how the binding words
31/// its errors is not documented, so anything else is a plain
32/// [`DecisionError::Backend`].
33const OUT_OF_CAPACITY: u32 = 3040;
34const DAILY_LIMIT: u32 = 3036;
35
36/// A SystemOne decision model run through a Worker's AI binding.
37///
38/// Holds the binding object (`env.AI`) and the catalog path of the model to
39/// run. What the binding throws or rejects with is classified like the REST
40/// API's errors:
41///
42/// - out of capacity (`3040`): [`DecisionError::RateLimited`], retried with
43///   [`with_retry`](Self::with_retry)'s policy (default
44///   [`RetryConfig::default`]);
45/// - daily free allocation used up (`3036`): [`DecisionError::Http`] 429, not
46///   retried — nothing succeeds before the reset;
47/// - anything else: [`DecisionError::Backend`] with the binding's message,
48///   not retried.
49///
50/// A [`DecisionModel::or`] fallback applies to all of them.
51#[derive(Clone, Debug)]
52pub struct AiBackend {
53    ai: JsValue,
54    model_path: String,
55    capabilities: Capabilities,
56    retry: RetryConfig,
57}
58
59impl AiBackend {
60    /// Run `model_path` (e.g. [`CLEF`]) through `ai`, the Worker's AI
61    /// binding: workers-rs's `Ai` (`env.ai("AI")?`) or the raw `env.AI`
62    /// `JsValue`.
63    ///
64    /// Capabilities: every question type, 255 options, 10 levels, batching,
65    /// and SystemOne's 64k/32k token estimates — a state over them fails
66    /// before the binding is called.
67    pub fn new(ai: impl Into<JsValue>, model_path: impl Into<String>) -> Self {
68        Self {
69            ai: ai.into(),
70            model_path: model_path.into(),
71            capabilities: Capabilities::new(QuestionKind::all())
72                .with_token_limits(Some(64_000), Some(32_000)),
73            retry: RetryConfig::default(),
74        }
75    }
76
77    /// Report these capabilities instead.
78    pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self {
79        self.capabilities = capabilities;
80        self
81    }
82
83    /// Retry policy for out-of-capacity errors (default
84    /// [`RetryConfig::default`]: 3 retries, 1 s initial, 2x, 30 s cap).
85    pub fn with_retry(mut self, retry: RetryConfig) -> Self {
86        self.retry = retry;
87        self
88    }
89
90    /// The catalog path this backend runs.
91    pub fn model_path(&self) -> &str {
92        &self.model_path
93    }
94
95    async fn run_once(&self, input: &JsValue) -> Result<serde_json::Value, DecisionError> {
96        let run = Reflect::get(&self.ai, &JsValue::from_str("run"))
97            .ok()
98            .and_then(|f| f.dyn_into::<Function>().ok())
99            .ok_or_else(|| {
100                DecisionError::Invalid(
101                    "the AI binding has no `run` method: pass the Worker's `env.AI`".into(),
102                )
103            })?;
104        // `run` may throw before returning its promise, or reject it.
105        let returned = run
106            .call2(&self.ai, &JsValue::from_str(&self.model_path), input)
107            .map_err(thrown)?;
108        let output = JsFuture::from(Promise::resolve(&returned))
109            .await
110            .map_err(thrown)?;
111        let text = JSON::stringify(&output)
112            .ok()
113            .and_then(|s| s.as_string())
114            .ok_or_else(|| {
115                DecisionError::BadResponse(format!(
116                    "the AI binding returned no JSON value: {}",
117                    describe(&output)
118                ))
119            })?;
120        serde_json::from_str(&text).map_err(|e| {
121            DecisionError::BadResponse(format!("the AI binding's output is not JSON: {e}"))
122        })
123    }
124}
125
126#[async_trait(?Send)]
127impl DecisionBackend for AiBackend {
128    fn capabilities(&self) -> Capabilities {
129        self.capabilities.clone()
130    }
131
132    async fn evaluate(&self, request: &Request) -> Result<Evaluation, DecisionError> {
133        let body = serde_json::to_string(request)
134            .map_err(|e| DecisionError::Invalid(format!("request does not serialize: {e}")))?;
135        let input = JSON::parse(&body).map_err(|e| {
136            DecisionError::Invalid(format!("request is not valid JSON: {}", describe(&e)))
137        })?;
138        let mut attempt = 0usize;
139        let value = loop {
140            match self.run_once(&input).await {
141                Err(e) if e.is_retryable() && attempt < self.retry.max_retries => {
142                    attempt += 1;
143                    yoagent::rt::sleep(self.retry.delay_for_attempt(attempt)).await;
144                }
145                other => break other?,
146            }
147        };
148        parse_systemone_response(value, request)
149    }
150}
151
152/// Clef through the Worker's AI binding: model `clef`, priced at the
153/// `cloudflare/clef` rate yoagent's process-wide price table holds when this
154/// is called — **unpriced unless the Worker opted in to prices first**
155/// ([`enable_bundled`](yoagent::provider::prices::enable_bundled): $0.24 per
156/// million input tokens). A later opt-in or
157/// [`install_override`](yoagent::provider::prices::global::install_override)
158/// does not reprice the returned model; with prices enabled but no rate in
159/// the table it is unpriced (logged).
160pub fn clef(ai: impl Into<JsValue>) -> DecisionModel {
161    model(ai, CLEF, "clef")
162}
163
164/// Clef Flash through the Worker's AI binding: model `clef-flash`, priced
165/// like [`clef`] at the `cloudflare/clef-flash` rate ($0.09 per million input
166/// tokens in the bundled snapshot; unpriced without an opt-in).
167pub fn clef_flash(ai: impl Into<JsValue>) -> DecisionModel {
168    model(ai, CLEF_FLASH, "clef-flash")
169}
170
171fn model(ai: impl Into<JsValue>, path: &str, id: &str) -> DecisionModel {
172    use yoagent::provider::prices::global;
173    let cost = global::resolved().cost(PRICE_PROVIDER, id);
174    // Unpriced is yoagent's default; a missing rate is news only once the
175    // Worker opted in to prices.
176    if cost.is_none() && global::pricing_enabled() {
177        tracing::warn!("no price for {PRICE_PROVIDER}/{id} in the price table; {id} is unpriced");
178    }
179    DecisionModel::from_backend(AiBackend::new(ai, path), id).with_cost(cost)
180}
181
182/// A binding failure, classified by its leading Workers AI error code.
183fn thrown(e: JsValue) -> DecisionError {
184    let message = format!("Workers AI: {}", describe(&e));
185    match leading_code(&message["Workers AI: ".len()..]) {
186        Some(OUT_OF_CAPACITY) => DecisionError::rate_limited_with_body(429, None, message),
187        Some(DAILY_LIMIT) => DecisionError::http(429, message),
188        _ => DecisionError::backend(message),
189    }
190}
191
192/// The `NNNN` of a message starting `NNNN:` (after an optional `Name: `).
193fn leading_code(message: &str) -> Option<u32> {
194    let rest = match message.split_once(": ") {
195        Some((name, rest)) if !name.starts_with(|c: char| c.is_ascii_digit()) => rest,
196        _ => message,
197    };
198    let (code, _) = rest.split_once(':')?;
199    (code.len() == 4).then(|| code.parse().ok()).flatten()
200}
201
202/// A JavaScript value as text: an `Error`'s name and message, a string, or
203/// its JSON.
204fn describe(v: &JsValue) -> String {
205    if let Some(e) = v.dyn_ref::<js_sys::Error>() {
206        let name = String::from(e.name());
207        let message = String::from(e.message());
208        return if name.is_empty() || name == "Error" {
209            message
210        } else {
211            format!("{name}: {message}")
212        };
213    }
214    if let Some(s) = v.as_string() {
215        return s;
216    }
217    JSON::stringify(v)
218        .ok()
219        .and_then(|s| s.as_string())
220        .unwrap_or_else(|| format!("{v:?}"))
221}