Skip to main content

drep/llm/client/
mod.rs

1//! The LLM client.
2//!
3//! One boundary: `LlmClient::complete_json` takes a system prompt and a user
4//! payload, sends them to the configured OpenAI-compatible endpoint, and
5//! returns the JSON value the model produced. Cache and concurrency limiting
6//! live above this single-provider request boundary.
7//!
8//! ## What is deliberately delegated to the SDK
9//!
10//! - **Streaming.** `open-agent-sdk` parses the SSE stream; drep concatenates
11//!   the `ContentBlock::Text` blocks it emits and ignores the rest. Since
12//!   0.10.0 those blocks are *fragments* - one event per delta, delivered
13//!   while the stream is open, where 0.9.x emitted the whole response as a
14//!   single block at the end. The types are identical either way, so nothing
15//!   here failed to compile and nothing here changed: the join in
16//!   `run_one_query` is what makes the assembled text independent of where
17//!   the deltas fall. Reading one block as the whole answer would now return a
18//!   prefix, and `src/llm/client/tests/streaming.rs` is what would notice.
19//! - **Transport retry.** `retry_with_backoff_conditional` decides per error
20//!   whether to retry (5xx, timeout, stream error) or fail fast (4xx, config
21//!   errors). drep adds no retry layer on top.
22//!
23//! ## What this module owns
24//!
25//! - **Parse retry.** The same prompt truncates the same way, so a parse
26//!   failure does NOT retry. The retry closure returns `Ok(None)` for an
27//!   unparseable body; the SDK's retry sees `Ok(...)` and stops.
28//! - **Attempt count floor.** `LlmConfig::max_retries` may be 0, but a
29//!   "zero attempts loop" would skip the request and report a bogus "no
30//!   exception was captured". The floor is 1.
31//! - **`max_tokens` pass-through.** The configured cap is forwarded only when
32//!   the user set one. open-agent-sdk 0.7.0 omits the field entirely otherwise,
33//!   so "unset" means the server decides - which is what a 256k-1M context
34//!   model needs. (Before 0.7.0 the builder substituted 4096 and truncated
35//!   reasoning models mid-thought; drep passed a large sentinel to work around
36//!   it. That workaround is gone.)
37
38use std::collections::BTreeMap;
39use std::time::Duration;
40
41use futures::StreamExt;
42use open_agent::retry::{RetryConfig, retry_with_backoff_conditional};
43use open_agent::{AgentOptions, ApiProtocol, ContentBlock, FinishReason, StreamEvent, query};
44
45use crate::config::LlmConfig;
46use crate::llm::error::LlmError;
47use crate::llm::json_parsing::{Extracted, extract_json};
48use crate::text::excerpt;
49
50/// How much of a model response reaches an error message.
51///
52/// Generous: unlike a URL, the useful signal in a refusal or a prose preamble
53/// is often a sentence or two in.
54const RESPONSE_EXCERPT_MAX: usize = 200;
55
56/// How much of a transport error reaches the report.
57///
58/// For an HTTP failure the SDK's error carries the endpoint's whole response
59/// body, read without a bound, and that text is the endpoint's rather than
60/// drep's. 400 matches what the Codex backend keeps of its own reported error,
61/// so the two backends' failure blocks read alike.
62const TRANSPORT_EXCERPT_MAX: usize = 400;
63
64/// A configured LLM client ready to issue requests.
65///
66/// Built once per process from `LlmConfig`; `complete_json` is the only
67/// entry point the analyzer uses.
68///
69/// Fields are `pub(crate)` so the test submodules can construct clients
70/// with a non-default retry config (the production default sleeps 1s
71/// between attempts, which would make the retry tests take seconds). They
72/// are not part of the public API.
73pub struct LlmClient {
74    pub(crate) base_url: String,
75    pub(crate) model: String,
76    pub(crate) api_key: String,
77    /// The wire protocol this endpoint speaks. Selects the request path, the auth
78    /// header, the body shape and the streaming vocabulary together - the SDK
79    /// resolves all four from this one value.
80    pub(crate) protocol: ApiProtocol,
81    /// `None` sends no `temperature` at all. Two of the four models drep ships a
82    /// preset for reject the parameter outright, and a 400 neither fails over nor
83    /// retries, so "omit it" had to be expressible rather than approximated by a
84    /// low value.
85    pub(crate) temperature: Option<f32>,
86    /// `None` means "no ceiling": since open-agent-sdk 0.7.0 an unset
87    /// `max_tokens` is omitted from the request entirely and the server decides.
88    /// Before 0.7.0 the builder substituted 4096, which truncated reasoning
89    /// models mid-thought, and drep had to pass a large sentinel instead.
90    pub(crate) max_tokens: Option<u32>,
91    pub(crate) timeout_secs: u64,
92    /// Exactly the headers this client sends, default included.
93    ///
94    /// The effective set rather than the configured one, resolved once in
95    /// [`crate::config::effective_headers`]. Applying the default inside the
96    /// request instead left three readers disagreeing about the same question:
97    /// a config naming one header printed one here and in `doctor` while the
98    /// request carried two, and a config naming none printed an empty set while
99    /// the request still carried a `User-Agent`. That is the divergence
100    /// `auth::source_of` exists to prevent one module over.
101    pub(crate) headers: BTreeMap<String, String>,
102    /// Precomputed because every reviewed file asks for the same provider's
103    /// cache identity. It may depend on credential-bearing header values, so it
104    /// is deliberately omitted from `Debug`.
105    request_identity: String,
106    pub(crate) retry_config: RetryConfig,
107}
108
109/// Hand-written so the API key cannot reach a log.
110///
111/// A derived `Debug` prints every field, so any `{:?}`, `dbg!` or tracing line
112/// touching the client would emit a live credential.
113impl std::fmt::Debug for LlmClient {
114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115        f.debug_struct("LlmClient")
116            .field("base_url", &self.base_url)
117            .field("model", &self.model)
118            .field("api_key", &"<redacted>")
119            .field("protocol", &self.protocol.as_str())
120            .field("temperature", &self.temperature)
121            .field("max_tokens", &self.max_tokens)
122            .field("timeout_secs", &self.timeout_secs)
123            // Names only, spelled exactly as `LlmConfig`'s `Debug` spells it. A
124            // header value is as likely to be a credential as `api_key` is, and
125            // this impl exists precisely so no `{:?}` emits one.
126            .field(
127                "headers",
128                &self.headers.keys().map(String::as_str).collect::<Vec<_>>(),
129            )
130            .finish()
131    }
132}
133
134impl LlmClient {
135    /// The model this client asks for.
136    ///
137    /// An accessor rather than a second copy on the caller: the cache key is
138    /// computed from the model, and a struct holding its own `model` string is
139    /// exactly what lets a request go to one model while the key names
140    /// another.
141    pub fn model(&self) -> &str {
142        &self.model
143    }
144
145    /// The base URL this client talks to. For display - `doctor` and the
146    /// failover report name the endpoint a provider used.
147    pub fn endpoint(&self) -> &str {
148        &self.base_url
149    }
150
151    /// The sampling temperature, or `None` when none is sent. Part of the cache
152    /// key, for the same reason [`Self::model`] is - and `None` has to key
153    /// differently from any value, because the answers genuinely differ.
154    pub fn temperature(&self) -> Option<f32> {
155        self.temperature
156    }
157
158    /// Every HTTP request option that can change the answer while the endpoint
159    /// and model stay the same.
160    ///
161    /// Header names are canonicalised to lower case because HTTP compares them
162    /// case-insensitively. Values remain exact: an arbitrary header can select a
163    /// tenant, model route or feature variant, and drep cannot infer from its
164    /// name whether changing it changes the answer. The returned digest is fed
165    /// into the cache hash and is never logged.
166    pub fn request_identity(&self) -> &str {
167        &self.request_identity
168    }
169
170    /// The wire protocol this client speaks. For display, and for the cache key:
171    /// the same model at the same endpoint over two protocols is two requests.
172    pub fn protocol(&self) -> ApiProtocol {
173        self.protocol
174    }
175
176    /// Build a client from a validated `LlmConfig`.
177    ///
178    /// Returns [`LlmError::NotConfigured`] when the config does not name an
179    /// endpoint, a model, or has `enabled = false`. We do not default an
180    /// endpoint - "LLM was disabled" and "LLM is enabled but misconfigured"
181    /// are both fatal here, and inventing a value would mask a broken
182    /// install.
183    pub fn new(cfg: &LlmConfig) -> Result<Self, LlmError> {
184        if !cfg.enabled {
185            return Err(LlmError::NotConfigured(
186                "LLM is disabled in config (set `enabled = true`)".to_string(),
187            ));
188        }
189        let endpoint = cfg.endpoint.clone().ok_or_else(|| {
190            LlmError::NotConfigured("LLM endpoint is not set in config".to_string())
191        })?;
192        let model = cfg
193            .model
194            .clone()
195            .ok_or_else(|| LlmError::NotConfigured("LLM model is not set in config".to_string()))?;
196
197        // The SDK deliberately omits protocol authentication for an empty key.
198        // This is what lets a gateway authenticate entirely through a custom
199        // header rather than receiving a fabricated `Bearer not-needed` or
200        // `x-api-key: not-needed` beside its real credential.
201        let api_key = cfg.api_key.clone().unwrap_or_default();
202
203        // `config::load` already rejected an unknown name, so this cannot fail for a
204        // config that came through the loader. It is re-checked rather than unwrapped
205        // because `LlmClient::new` is also reachable from tests that build an
206        // `LlmConfig` directly, and a silent default here would post
207        // chat-completions bytes to a `/messages` endpoint.
208        let protocol = crate::config::parse_protocol(cfg.protocol.as_deref()).ok_or_else(|| {
209            LlmError::NotConfigured(format!(
210                "unknown protocol `{}`; expected `openai` or `anthropic`",
211                cfg.protocol.as_deref().unwrap_or_default()
212            ))
213        })?;
214
215        // max_retries is a total attempt count; the floor is 1 so a config of
216        // 0 still performs exactly one attempt. See the spec's note on the
217        // bogus "no exception was captured" failure.
218        let max_attempts = cfg.max_retries.max(1);
219
220        let headers = crate::config::effective_headers(&cfg.headers);
221        let request_identity = build_request_identity(protocol, cfg.max_tokens, &headers)?;
222
223        Ok(LlmClient {
224            base_url: endpoint,
225            model,
226            api_key,
227            protocol,
228            temperature: cfg.temperature,
229            max_tokens: cfg.max_tokens,
230            timeout_secs: cfg.timeout_secs,
231            headers,
232            request_identity,
233            retry_config: RetryConfig {
234                max_attempts,
235                initial_delay: Duration::from_secs(1),
236                max_delay: Duration::from_secs(60),
237                backoff_multiplier: 2.0,
238                jitter_factor: 0.1,
239            },
240        })
241    }
242
243    /// Send one prompt and return the extracted JSON.
244    ///
245    /// Concatenates `ContentBlock::Text` blocks in arrival order; other block
246    /// variants are ignored. An empty response body is **retried** as a
247    /// transport failure and, if it keeps coming back empty, surfaces as
248    /// [`LlmError::Transport`] - see `run_one_query` for why "the model
249    /// returned nothing" is not the deterministic outcome it looks like.
250    ///
251    /// A non-empty body that yields **no JSON at all** is retried up to
252    /// [`NO_JSON_ATTEMPTS`] times and then becomes [`LlmError::Unparseable`],
253    /// carrying an excerpt of what actually came back. A body that parsed only
254    /// after brace-balancing ([`Extracted::Truncated`]) is returned
255    /// immediately and never retried - that is the genuinely deterministic
256    /// case, and the one the "never retry" rule was written for.
257    pub async fn complete_json(
258        &self,
259        system_prompt: &str,
260        user_content: &str,
261    ) -> Result<Extracted, LlmError> {
262        // Build options per request. The SDK doesn't expose a way to set
263        // `system_prompt` after `build()`, and the retry closure needs to
264        // borrow the same options across attempts.
265        let builder = AgentOptions::builder()
266            .model(&self.model)
267            .base_url(&self.base_url)
268            .api_key(&self.api_key)
269            .system_prompt(system_prompt)
270            .protocol(self.protocol)
271            .timeout(self.timeout_secs);
272
273        // Only send a temperature when one was configured. An unset value means the
274        // field is omitted entirely, which is the only thing that works against a
275        // model that rejects the parameter.
276        let builder = match self.temperature {
277            Some(temperature) => builder.temperature(temperature),
278            None => builder,
279        };
280
281        // Only set a ceiling when the user asked for one. open-agent-sdk 0.7.0
282        // omits `max_tokens` from the request when the setter is never called,
283        // so "unset" genuinely means "let the server decide" rather than the
284        // implicit 4096 earlier versions substituted.
285        let builder = match self.max_tokens {
286            Some(limit) => builder.max_tokens(limit),
287            None => builder,
288        };
289
290        // No ordering rule here, and that is the point: `self.headers` is
291        // already the effective set, resolved once when the client was built, so
292        // this loop cannot get the precedence between drep's own `User-Agent`
293        // and an operator's replacement for it wrong.
294        let mut builder = builder;
295        for (name, value) in &self.headers {
296            builder = builder.header(name, value);
297        }
298
299        let options = builder
300            .build()
301            .map_err(|e| LlmError::NotConfigured(format!("AgentOptions build failed: {e}")))?;
302
303        let prompt = user_content.to_string();
304
305        // The no-JSON retry is drep's own loop, deliberately *outside* the
306        // SDK's. Handing "no JSON" to the SDK by returning `Err` would work,
307        // but it would surface as `LlmError::Transport` once the attempts ran
308        // out - and `Transport` fails over to the next provider and demotes
309        // this one for the whole run. A model that answered in prose has told
310        // us nothing about the endpoint: after these response retries the
311        // chain may ask a fallback for this file, but must not demote this
312        // provider.
313        //
314        // The SDK's own retry still runs inside each pass, so a transport
315        // failure is handled by the layer that classifies it.
316        let mut last_body = String::new();
317        for _ in 0..NO_JSON_ATTEMPTS {
318            let result: open_agent::Result<Answer> =
319                retry_with_backoff_conditional(self.retry_config.clone(), || {
320                    self.run_one_query(&prompt, &options)
321                })
322                .await;
323
324            match result {
325                Ok(Answer::Parsed(extracted)) => return Ok(extracted),
326                // The server said why it stopped, and the reason rules out a
327                // retry: the request hit a limit, so the same request hits the
328                // same limit. This is the genuinely deterministic case the
329                // original "never retry a non-empty body" rule was reaching
330                // for - it just used "no JSON in the body" as the proxy, which
331                // is not the same question.
332                Ok(Answer::NoJson { text, finish }) if !worth_asking_again(&finish) => {
333                    return Err(LlmError::ModelStopped {
334                        finish: finish.as_str().to_owned(),
335                        message: stopped_message(&finish, &text),
336                    });
337                }
338                Ok(Answer::NoJson { text, .. }) => last_body = text,
339                Err(e) => {
340                    // The SDK exposes the status code separately (via
341                    // `status_code`); reading it before formatting means the
342                    // number survives as a number, and a later caller can
343                    // branch on it rather than parsing the message.
344                    //
345                    // The message goes through `excerpt` like every other
346                    // piece of endpoint text. For a non-success status it is
347                    // the response body verbatim: a hostile or compromised
348                    // endpoint chooses every byte, and the failure block
349                    // prints it to the terminal, once for every file the
350                    // demoted provider then skips.
351                    let status = e.status_code();
352                    let message = excerpt(&e.to_string(), TRANSPORT_EXCERPT_MAX);
353                    return Err(LlmError::Transport { status, message });
354                }
355            }
356        }
357
358        Err(LlmError::Unparseable(format!(
359            "no JSON in the response after {NO_JSON_ATTEMPTS} attempts; \
360             the model answered: {}",
361            excerpt(&last_body, RESPONSE_EXCERPT_MAX)
362        )))
363    }
364
365    /// One attempt: stream the response, concatenate text, parse.
366    ///
367    /// Returns [`Answer::NoJson`] carrying the raw text when the query
368    /// produced something we could not parse at all - the SDK's retry layer
369    /// sees `Ok` and stops, leaving the decision to `complete_json`. Returns
370    /// `Err(SdkError)` for a transport-level failure, including an unexplained
371    /// empty response. An empty response with a terminal `Length` or
372    /// `ContentFilter` reason stays [`Answer::NoJson`], because the reason says
373    /// the same request cannot benefit from a retry.
374    async fn run_one_query(
375        &self,
376        prompt: &str,
377        options: &AgentOptions,
378    ) -> open_agent::Result<Answer> {
379        let mut stream = query(prompt, options).await?;
380        let mut text = String::new();
381        // `Unspecified` is the right default rather than a panic-if-absent:
382        // several OpenAI-compatible servers never report a reason at all, and
383        // "no information" is a distinct answer from "stopped normally".
384        let mut finish = FinishReason::Unspecified;
385        while let Some(event) = stream.next().await {
386            match event? {
387                // Image, ToolUse, ToolResult are not used here.
388                StreamEvent::Block(ContentBlock::Text(t)) => text.push_str(&t.text),
389                StreamEvent::Finish(reason) => finish = reason,
390                // Everything else is discarded, and that is the contract:
391                // `text` holds assistant text and nothing else. It covers the
392                // non-text blocks drep has no use for, the `Reasoning` side
393                // channel (opt-in, and drep does not opt in - chain-of-thought
394                // must never reach the text drep parses as JSON), and any
395                // variant a later SDK adds, since `StreamEvent` is
396                // `#[non_exhaustive]`. Spelled as one arm because a separate
397                // `Reasoning(_) => {}` above it does the same nothing, and an
398                // arm indistinguishable from the wildcard is dead code.
399                _ => {}
400            }
401        }
402
403        // An empty body is a **transport** failure, not a parse failure.
404        //
405        // An empty response is provider flakiness, not a deterministic parse
406        // failure for the prompt. Repeating the same request can immediately
407        // succeed with findings.
408        //
409        // `Error::stream` is classified retryable by the SDK, which is both
410        // accurate (the stream completed carrying no content) and nearly free:
411        // a response with no output tokens cost nothing to produce, so asking
412        // again is cheap. A *non-empty* body we cannot parse still returns
413        // `Ok(None)` and still does not retry - that is the deterministic case
414        // the split was built for, and re-sending it burns a full reasoning
415        // call for the same answer.
416        if text.trim().is_empty() && worth_asking_again(&finish) {
417            return Err(open_agent::Error::stream(
418                "the model returned an empty response",
419            ));
420        }
421
422        Ok(match extract_json(&text) {
423            Some(extracted) => Answer::Parsed(extracted),
424            // The text is carried out rather than dropped. It was discarded
425            // behind the constant "response contained no parseable JSON",
426            // which made every occurrence of this failure look identical and
427            // left no way to tell a refusal from a prose preamble from
428            // reasoning that leaked into the content channel.
429            None => Answer::NoJson { text, finish },
430        })
431    }
432}
433
434/// Build the fixed digest nested inside the response-cache key.
435///
436/// Length-prefixing through the cache's own encoder handles arbitrary UTF-8
437/// header values without reserving an escaping byte. Only the digest is kept;
438/// the framed credential-bearing text is never stored or logged.
439fn build_request_identity(
440    protocol: ApiProtocol,
441    max_tokens: Option<u32>,
442    headers: &BTreeMap<String, String>,
443) -> Result<String, LlmError> {
444    use crate::llm::cache::write_field;
445
446    let mut identity = blake3::Hasher::new();
447    write_field(&mut identity, protocol.as_str().as_bytes());
448    match max_tokens {
449        Some(limit) => {
450            write_field(&mut identity, b"set");
451            write_field(&mut identity, &limit.to_be_bytes());
452        }
453        None => write_field(&mut identity, b"unset"),
454    }
455
456    let mut canonical_headers = headers
457        .iter()
458        .map(|(name, value)| {
459            let canonical =
460                reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| {
461                    LlmError::NotConfigured(format!("header name `{name}` cannot be encoded"))
462                })?;
463            Ok((canonical, value))
464        })
465        .collect::<Result<Vec<_>, LlmError>>()?;
466    canonical_headers.sort_by(|left, right| left.0.as_str().cmp(right.0.as_str()));
467    let header_count = u64::try_from(canonical_headers.len())
468        .expect("the process cannot hold more than u64::MAX headers");
469    write_field(&mut identity, &header_count.to_be_bytes());
470    for (name, value) in canonical_headers {
471        write_field(&mut identity, name.as_str().as_bytes());
472        write_field(&mut identity, value.as_bytes());
473    }
474    Ok(identity.finalize().to_hex().to_string())
475}
476
477/// What one query produced, before the retry decision.
478///
479/// `NoJson` carries the body so the failure can be diagnosed and so
480/// `complete_json` can decide whether to ask again. The SDK's retry layer
481/// treats both variants as success and stops, which is what keeps the
482/// no-JSON decision here rather than inside it.
483enum Answer {
484    Parsed(Extracted),
485    /// No JSON at all, with why generation stopped. The reason decides whether
486    /// asking again can possibly help.
487    NoJson {
488        text: String,
489        finish: FinishReason,
490    },
491}
492
493/// How many times a response carrying no JSON at all is asked for again.
494///
495/// Not the same question as the SDK's transport retry, and deliberately a
496/// small number: each attempt is a full reasoning call. The rule this replaced
497/// never retried, justified as "the same prompt truncates the same way" - but
498/// that is [`Extracted::Truncated`], a different branch. A response with *no
499/// JSON at all* did not truncate an answer, it never produced one, and in
500/// practice it does not repeat: drep's own gated push failed on a different
501/// file each run, and each failing file analyzed cleanly when asked again.
502///
503/// Three total attempts, so two local retries before the provider chain may
504/// ask a fallback. Production output has been visibly garbled twice in a row
505/// and then parsed unchanged on a later run; the third attempt salvages that
506/// case without demoting an otherwise healthy provider.
507pub const NO_JSON_ATTEMPTS: u32 = 3;
508
509/// Whether asking the same question again could produce a different answer.
510///
511/// `false` for the reasons that are a property of the *request*: a token cap is
512/// hit identically every time, and a content filter that refused this payload
513/// refuses it again. `true` where the server told us nothing useful, because a
514/// model at temperature above zero can simply answer differently - which is
515/// what drep's own gated push demonstrated, failing on a different file each
516/// run with every failing file analyzing cleanly when asked again.
517fn worth_asking_again(finish: &FinishReason) -> bool {
518    // Written as a negated match on the two request-shaped reasons rather than
519    // as an enumeration of the rest. `FinishReason` is `#[non_exhaustive]`, so
520    // a wildcard arm is required either way - and an enumerated "everything
521    // else is retryable" arm sitting above it is behaviourally identical to the
522    // wildcard, which makes it undeletable-but-unobservable: exactly the dead
523    // code the mutation gate exists to find.
524    //
525    // The consequence of the wildcard is deliberate: a reason a later SDK adds
526    // defaults to retrying. The retry is bounded and cheap to be wrong about,
527    // whereas refusing to retry something transient fails a commit outright.
528    !matches!(
529        finish,
530        // A token cap is hit identically every time - drep sends no
531        // `max_tokens`, so the cap is the server's. A content filter that
532        // refused this payload refuses it again.
533        FinishReason::Length | FinishReason::ContentFilter
534    )
535}
536
537/// A sentence a user can act on, for the reasons that end the attempt.
538///
539/// The two cases want different actions - one is "this file is too big for this
540/// model in one pass", the other is "this provider refused the content" - so
541/// they do not share a message.
542fn stopped_message(finish: &FinishReason, text: &str) -> String {
543    match finish {
544        FinishReason::Length => format!(
545            "the model hit its output token limit before producing any JSON. \
546             This file is too large for this model to review in one request - \
547             split it, or use a provider with a larger output budget. \
548             It managed: {}",
549            excerpt(text, RESPONSE_EXCERPT_MAX)
550        ),
551        _ => format!(
552            "the model stopped ({}) before producing any JSON: {}",
553            finish.as_str(),
554            excerpt(text, RESPONSE_EXCERPT_MAX)
555        ),
556    }
557}
558
559#[cfg(test)]
560mod tests;