drep/llm/client/mod.rs
1//! The LLM client.
2//!
3//! One boundary: `LlmClient::complete_json` takes a system prompt and a user
4//! payload, sends them to the configured OpenAI-compatible endpoint, and
5//! returns the JSON value the model produced. Cache and concurrency limiting
6//! live above this single-provider request boundary.
7//!
8//! ## What is deliberately delegated to the SDK
9//!
10//! - **Streaming.** `open-agent-sdk` parses the SSE stream; drep concatenates
11//! the `ContentBlock::Text` blocks it emits and ignores the rest. Since
12//! 0.10.0 those blocks are *fragments* - one event per delta, delivered
13//! while the stream is open, where 0.9.x emitted the whole response as a
14//! single block at the end. The types are identical either way, so nothing
15//! here failed to compile and nothing here changed: the join in
16//! `run_one_query` is what makes the assembled text independent of where
17//! the deltas fall. Reading one block as the whole answer would now return a
18//! prefix, and `src/llm/client/tests/streaming.rs` is what would notice.
19//! - **Transport retry.** `retry_with_backoff_conditional` decides per error
20//! whether to retry (5xx, timeout, stream error) or fail fast (4xx, config
21//! errors). drep adds no retry layer on top.
22//!
23//! ## What this module owns
24//!
25//! - **Parse retry.** The same prompt truncates the same way, so a parse
26//! failure does NOT retry. The retry closure returns `Ok(None)` for an
27//! unparseable body; the SDK's retry sees `Ok(...)` and stops.
28//! - **Attempt count floor.** `LlmConfig::max_retries` may be 0, but a
29//! "zero attempts loop" would skip the request and report a bogus "no
30//! exception was captured". The floor is 1.
31//! - **`max_tokens` pass-through.** The configured cap is forwarded only when
32//! the user set one. open-agent-sdk 0.7.0 omits the field entirely otherwise,
33//! so "unset" means the server decides - which is what a 256k-1M context
34//! model needs. (Before 0.7.0 the builder substituted 4096 and truncated
35//! reasoning models mid-thought; drep passed a large sentinel to work around
36//! it. That workaround is gone.)
37
38use std::collections::BTreeMap;
39use std::time::Duration;
40
41use futures::StreamExt;
42use open_agent::retry::{RetryConfig, retry_with_backoff_conditional};
43use open_agent::{AgentOptions, ApiProtocol, ContentBlock, FinishReason, StreamEvent, query};
44
45use crate::config::LlmConfig;
46use crate::llm::error::LlmError;
47use crate::llm::json_parsing::{Extracted, extract_json};
48use crate::text::excerpt;
49
50/// How much of a model response reaches an error message.
51///
52/// Generous: unlike a URL, the useful signal in a refusal or a prose preamble
53/// is often a sentence or two in.
54const RESPONSE_EXCERPT_MAX: usize = 200;
55
56/// How much of a transport error reaches the report.
57///
58/// For an HTTP failure the SDK's error carries the endpoint's whole response
59/// body, read without a bound, and that text is the endpoint's rather than
60/// drep's. 400 matches what the Codex backend keeps of its own reported error,
61/// so the two backends' failure blocks read alike.
62const TRANSPORT_EXCERPT_MAX: usize = 400;
63
64/// A configured LLM client ready to issue requests.
65///
66/// Built once per process from `LlmConfig`; `complete_json` is the only
67/// entry point the analyzer uses.
68///
69/// Fields are `pub(crate)` so the test submodules can construct clients
70/// with a non-default retry config (the production default sleeps 1s
71/// between attempts, which would make the retry tests take seconds). They
72/// are not part of the public API.
73pub struct LlmClient {
74 pub(crate) base_url: String,
75 pub(crate) model: String,
76 pub(crate) api_key: String,
77 /// The wire protocol this endpoint speaks. Selects the request path, the auth
78 /// header, the body shape and the streaming vocabulary together - the SDK
79 /// resolves all four from this one value.
80 pub(crate) protocol: ApiProtocol,
81 /// `None` sends no `temperature` at all. Two of the four models drep ships a
82 /// preset for reject the parameter outright, and a 400 neither fails over nor
83 /// retries, so "omit it" had to be expressible rather than approximated by a
84 /// low value.
85 pub(crate) temperature: Option<f32>,
86 /// `None` means "no ceiling": since open-agent-sdk 0.7.0 an unset
87 /// `max_tokens` is omitted from the request entirely and the server decides.
88 /// Before 0.7.0 the builder substituted 4096, which truncated reasoning
89 /// models mid-thought, and drep had to pass a large sentinel instead.
90 pub(crate) max_tokens: Option<u32>,
91 pub(crate) timeout_secs: u64,
92 /// Exactly the headers this client sends, default included.
93 ///
94 /// The effective set rather than the configured one, resolved once in
95 /// [`crate::config::effective_headers`]. Applying the default inside the
96 /// request instead left three readers disagreeing about the same question:
97 /// a config naming one header printed one here and in `doctor` while the
98 /// request carried two, and a config naming none printed an empty set while
99 /// the request still carried a `User-Agent`. That is the divergence
100 /// `auth::source_of` exists to prevent one module over.
101 pub(crate) headers: BTreeMap<String, String>,
102 /// Precomputed because every reviewed file asks for the same provider's
103 /// cache identity. It may depend on credential-bearing header values, so it
104 /// is deliberately omitted from `Debug`.
105 request_identity: String,
106 pub(crate) retry_config: RetryConfig,
107}
108
109/// Hand-written so the API key cannot reach a log.
110///
111/// A derived `Debug` prints every field, so any `{:?}`, `dbg!` or tracing line
112/// touching the client would emit a live credential.
113impl std::fmt::Debug for LlmClient {
114 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115 f.debug_struct("LlmClient")
116 .field("base_url", &self.base_url)
117 .field("model", &self.model)
118 .field("api_key", &"<redacted>")
119 .field("protocol", &self.protocol.as_str())
120 .field("temperature", &self.temperature)
121 .field("max_tokens", &self.max_tokens)
122 .field("timeout_secs", &self.timeout_secs)
123 // Names only, spelled exactly as `LlmConfig`'s `Debug` spells it. A
124 // header value is as likely to be a credential as `api_key` is, and
125 // this impl exists precisely so no `{:?}` emits one.
126 .field(
127 "headers",
128 &self.headers.keys().map(String::as_str).collect::<Vec<_>>(),
129 )
130 .finish()
131 }
132}
133
134impl LlmClient {
135 /// The model this client asks for.
136 ///
137 /// An accessor rather than a second copy on the caller: the cache key is
138 /// computed from the model, and a struct holding its own `model` string is
139 /// exactly what lets a request go to one model while the key names
140 /// another.
141 pub fn model(&self) -> &str {
142 &self.model
143 }
144
145 /// The base URL this client talks to. For display - `doctor` and the
146 /// failover report name the endpoint a provider used.
147 pub fn endpoint(&self) -> &str {
148 &self.base_url
149 }
150
151 /// The sampling temperature, or `None` when none is sent. Part of the cache
152 /// key, for the same reason [`Self::model`] is - and `None` has to key
153 /// differently from any value, because the answers genuinely differ.
154 pub fn temperature(&self) -> Option<f32> {
155 self.temperature
156 }
157
158 /// Every HTTP request option that can change the answer while the endpoint
159 /// and model stay the same.
160 ///
161 /// Header names are canonicalised to lower case because HTTP compares them
162 /// case-insensitively. Values remain exact: an arbitrary header can select a
163 /// tenant, model route or feature variant, and drep cannot infer from its
164 /// name whether changing it changes the answer. The returned digest is fed
165 /// into the cache hash and is never logged.
166 pub fn request_identity(&self) -> &str {
167 &self.request_identity
168 }
169
170 /// The wire protocol this client speaks. For display, and for the cache key:
171 /// the same model at the same endpoint over two protocols is two requests.
172 pub fn protocol(&self) -> ApiProtocol {
173 self.protocol
174 }
175
176 /// Build a client from a validated `LlmConfig`.
177 ///
178 /// Returns [`LlmError::NotConfigured`] when the config does not name an
179 /// endpoint, a model, or has `enabled = false`. We do not default an
180 /// endpoint - "LLM was disabled" and "LLM is enabled but misconfigured"
181 /// are both fatal here, and inventing a value would mask a broken
182 /// install.
183 pub fn new(cfg: &LlmConfig) -> Result<Self, LlmError> {
184 if !cfg.enabled {
185 return Err(LlmError::NotConfigured(
186 "LLM is disabled in config (set `enabled = true`)".to_string(),
187 ));
188 }
189 let endpoint = cfg.endpoint.clone().ok_or_else(|| {
190 LlmError::NotConfigured("LLM endpoint is not set in config".to_string())
191 })?;
192 let model = cfg
193 .model
194 .clone()
195 .ok_or_else(|| LlmError::NotConfigured("LLM model is not set in config".to_string()))?;
196
197 // The SDK deliberately omits protocol authentication for an empty key.
198 // This is what lets a gateway authenticate entirely through a custom
199 // header rather than receiving a fabricated `Bearer not-needed` or
200 // `x-api-key: not-needed` beside its real credential.
201 let api_key = cfg.api_key.clone().unwrap_or_default();
202
203 // `config::load` already rejected an unknown name, so this cannot fail for a
204 // config that came through the loader. It is re-checked rather than unwrapped
205 // because `LlmClient::new` is also reachable from tests that build an
206 // `LlmConfig` directly, and a silent default here would post
207 // chat-completions bytes to a `/messages` endpoint.
208 let protocol = crate::config::parse_protocol(cfg.protocol.as_deref()).ok_or_else(|| {
209 LlmError::NotConfigured(format!(
210 "unknown protocol `{}`; expected `openai` or `anthropic`",
211 cfg.protocol.as_deref().unwrap_or_default()
212 ))
213 })?;
214
215 // max_retries is a total attempt count; the floor is 1 so a config of
216 // 0 still performs exactly one attempt. See the spec's note on the
217 // bogus "no exception was captured" failure.
218 let max_attempts = cfg.max_retries.max(1);
219
220 let headers = crate::config::effective_headers(&cfg.headers);
221 let request_identity = build_request_identity(protocol, cfg.max_tokens, &headers)?;
222
223 Ok(LlmClient {
224 base_url: endpoint,
225 model,
226 api_key,
227 protocol,
228 temperature: cfg.temperature,
229 max_tokens: cfg.max_tokens,
230 timeout_secs: cfg.timeout_secs,
231 headers,
232 request_identity,
233 retry_config: RetryConfig {
234 max_attempts,
235 initial_delay: Duration::from_secs(1),
236 max_delay: Duration::from_secs(60),
237 backoff_multiplier: 2.0,
238 jitter_factor: 0.1,
239 },
240 })
241 }
242
243 /// Send one prompt and return the extracted JSON.
244 ///
245 /// Concatenates `ContentBlock::Text` blocks in arrival order; other block
246 /// variants are ignored. An empty response body is **retried** as a
247 /// transport failure and, if it keeps coming back empty, surfaces as
248 /// [`LlmError::Transport`] - see `run_one_query` for why "the model
249 /// returned nothing" is not the deterministic outcome it looks like.
250 ///
251 /// A non-empty body that yields **no JSON at all** is retried up to
252 /// [`NO_JSON_ATTEMPTS`] times and then becomes [`LlmError::Unparseable`],
253 /// carrying an excerpt of what actually came back. A body that parsed only
254 /// after brace-balancing ([`Extracted::Truncated`]) is returned
255 /// immediately and never retried - that is the genuinely deterministic
256 /// case, and the one the "never retry" rule was written for.
257 pub async fn complete_json(
258 &self,
259 system_prompt: &str,
260 user_content: &str,
261 ) -> Result<Extracted, LlmError> {
262 // Build options per request. The SDK doesn't expose a way to set
263 // `system_prompt` after `build()`, and the retry closure needs to
264 // borrow the same options across attempts.
265 let builder = AgentOptions::builder()
266 .model(&self.model)
267 .base_url(&self.base_url)
268 .api_key(&self.api_key)
269 .system_prompt(system_prompt)
270 .protocol(self.protocol)
271 .timeout(self.timeout_secs);
272
273 // Only send a temperature when one was configured. An unset value means the
274 // field is omitted entirely, which is the only thing that works against a
275 // model that rejects the parameter.
276 let builder = match self.temperature {
277 Some(temperature) => builder.temperature(temperature),
278 None => builder,
279 };
280
281 // Only set a ceiling when the user asked for one. open-agent-sdk 0.7.0
282 // omits `max_tokens` from the request when the setter is never called,
283 // so "unset" genuinely means "let the server decide" rather than the
284 // implicit 4096 earlier versions substituted.
285 let builder = match self.max_tokens {
286 Some(limit) => builder.max_tokens(limit),
287 None => builder,
288 };
289
290 // No ordering rule here, and that is the point: `self.headers` is
291 // already the effective set, resolved once when the client was built, so
292 // this loop cannot get the precedence between drep's own `User-Agent`
293 // and an operator's replacement for it wrong.
294 let mut builder = builder;
295 for (name, value) in &self.headers {
296 builder = builder.header(name, value);
297 }
298
299 let options = builder
300 .build()
301 .map_err(|e| LlmError::NotConfigured(format!("AgentOptions build failed: {e}")))?;
302
303 let prompt = user_content.to_string();
304
305 // The no-JSON retry is drep's own loop, deliberately *outside* the
306 // SDK's. Handing "no JSON" to the SDK by returning `Err` would work,
307 // but it would surface as `LlmError::Transport` once the attempts ran
308 // out - and `Transport` fails over to the next provider and demotes
309 // this one for the whole run. A model that answered in prose has told
310 // us nothing about the endpoint: after these response retries the
311 // chain may ask a fallback for this file, but must not demote this
312 // provider.
313 //
314 // The SDK's own retry still runs inside each pass, so a transport
315 // failure is handled by the layer that classifies it.
316 let mut last_body = String::new();
317 for _ in 0..NO_JSON_ATTEMPTS {
318 let result: open_agent::Result<Answer> =
319 retry_with_backoff_conditional(self.retry_config.clone(), || {
320 self.run_one_query(&prompt, &options)
321 })
322 .await;
323
324 match result {
325 Ok(Answer::Parsed(extracted)) => return Ok(extracted),
326 // The server said why it stopped, and the reason rules out a
327 // retry: the request hit a limit, so the same request hits the
328 // same limit. This is the genuinely deterministic case the
329 // original "never retry a non-empty body" rule was reaching
330 // for - it just used "no JSON in the body" as the proxy, which
331 // is not the same question.
332 Ok(Answer::NoJson { text, finish }) if !worth_asking_again(&finish) => {
333 return Err(LlmError::ModelStopped {
334 finish: finish.as_str().to_owned(),
335 message: stopped_message(&finish, &text),
336 });
337 }
338 Ok(Answer::NoJson { text, .. }) => last_body = text,
339 Err(e) => {
340 // The SDK exposes the status code separately (via
341 // `status_code`); reading it before formatting means the
342 // number survives as a number, and a later caller can
343 // branch on it rather than parsing the message.
344 //
345 // The message goes through `excerpt` like every other
346 // piece of endpoint text. For a non-success status it is
347 // the response body verbatim: a hostile or compromised
348 // endpoint chooses every byte, and the failure block
349 // prints it to the terminal, once for every file the
350 // demoted provider then skips.
351 let status = e.status_code();
352 let message = excerpt(&e.to_string(), TRANSPORT_EXCERPT_MAX);
353 return Err(LlmError::Transport { status, message });
354 }
355 }
356 }
357
358 Err(LlmError::Unparseable(format!(
359 "no JSON in the response after {NO_JSON_ATTEMPTS} attempts; \
360 the model answered: {}",
361 excerpt(&last_body, RESPONSE_EXCERPT_MAX)
362 )))
363 }
364
365 /// One attempt: stream the response, concatenate text, parse.
366 ///
367 /// Returns [`Answer::NoJson`] carrying the raw text when the query
368 /// produced something we could not parse at all - the SDK's retry layer
369 /// sees `Ok` and stops, leaving the decision to `complete_json`. Returns
370 /// `Err(SdkError)` for a transport-level failure, including an unexplained
371 /// empty response. An empty response with a terminal `Length` or
372 /// `ContentFilter` reason stays [`Answer::NoJson`], because the reason says
373 /// the same request cannot benefit from a retry.
374 async fn run_one_query(
375 &self,
376 prompt: &str,
377 options: &AgentOptions,
378 ) -> open_agent::Result<Answer> {
379 let mut stream = query(prompt, options).await?;
380 let mut text = String::new();
381 // `Unspecified` is the right default rather than a panic-if-absent:
382 // several OpenAI-compatible servers never report a reason at all, and
383 // "no information" is a distinct answer from "stopped normally".
384 let mut finish = FinishReason::Unspecified;
385 while let Some(event) = stream.next().await {
386 match event? {
387 // Image, ToolUse, ToolResult are not used here.
388 StreamEvent::Block(ContentBlock::Text(t)) => text.push_str(&t.text),
389 StreamEvent::Finish(reason) => finish = reason,
390 // Everything else is discarded, and that is the contract:
391 // `text` holds assistant text and nothing else. It covers the
392 // non-text blocks drep has no use for, the `Reasoning` side
393 // channel (opt-in, and drep does not opt in - chain-of-thought
394 // must never reach the text drep parses as JSON), and any
395 // variant a later SDK adds, since `StreamEvent` is
396 // `#[non_exhaustive]`. Spelled as one arm because a separate
397 // `Reasoning(_) => {}` above it does the same nothing, and an
398 // arm indistinguishable from the wildcard is dead code.
399 _ => {}
400 }
401 }
402
403 // An empty body is a **transport** failure, not a parse failure.
404 //
405 // An empty response is provider flakiness, not a deterministic parse
406 // failure for the prompt. Repeating the same request can immediately
407 // succeed with findings.
408 //
409 // `Error::stream` is classified retryable by the SDK, which is both
410 // accurate (the stream completed carrying no content) and nearly free:
411 // a response with no output tokens cost nothing to produce, so asking
412 // again is cheap. A *non-empty* body we cannot parse still returns
413 // `Ok(None)` and still does not retry - that is the deterministic case
414 // the split was built for, and re-sending it burns a full reasoning
415 // call for the same answer.
416 if text.trim().is_empty() && worth_asking_again(&finish) {
417 return Err(open_agent::Error::stream(
418 "the model returned an empty response",
419 ));
420 }
421
422 Ok(match extract_json(&text) {
423 Some(extracted) => Answer::Parsed(extracted),
424 // The text is carried out rather than dropped. It was discarded
425 // behind the constant "response contained no parseable JSON",
426 // which made every occurrence of this failure look identical and
427 // left no way to tell a refusal from a prose preamble from
428 // reasoning that leaked into the content channel.
429 None => Answer::NoJson { text, finish },
430 })
431 }
432}
433
434/// Build the fixed digest nested inside the response-cache key.
435///
436/// Length-prefixing through the cache's own encoder handles arbitrary UTF-8
437/// header values without reserving an escaping byte. Only the digest is kept;
438/// the framed credential-bearing text is never stored or logged.
439fn build_request_identity(
440 protocol: ApiProtocol,
441 max_tokens: Option<u32>,
442 headers: &BTreeMap<String, String>,
443) -> Result<String, LlmError> {
444 use crate::llm::cache::write_field;
445
446 let mut identity = blake3::Hasher::new();
447 write_field(&mut identity, protocol.as_str().as_bytes());
448 match max_tokens {
449 Some(limit) => {
450 write_field(&mut identity, b"set");
451 write_field(&mut identity, &limit.to_be_bytes());
452 }
453 None => write_field(&mut identity, b"unset"),
454 }
455
456 let mut canonical_headers = headers
457 .iter()
458 .map(|(name, value)| {
459 let canonical =
460 reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| {
461 LlmError::NotConfigured(format!("header name `{name}` cannot be encoded"))
462 })?;
463 Ok((canonical, value))
464 })
465 .collect::<Result<Vec<_>, LlmError>>()?;
466 canonical_headers.sort_by(|left, right| left.0.as_str().cmp(right.0.as_str()));
467 let header_count = u64::try_from(canonical_headers.len())
468 .expect("the process cannot hold more than u64::MAX headers");
469 write_field(&mut identity, &header_count.to_be_bytes());
470 for (name, value) in canonical_headers {
471 write_field(&mut identity, name.as_str().as_bytes());
472 write_field(&mut identity, value.as_bytes());
473 }
474 Ok(identity.finalize().to_hex().to_string())
475}
476
477/// What one query produced, before the retry decision.
478///
479/// `NoJson` carries the body so the failure can be diagnosed and so
480/// `complete_json` can decide whether to ask again. The SDK's retry layer
481/// treats both variants as success and stops, which is what keeps the
482/// no-JSON decision here rather than inside it.
483enum Answer {
484 Parsed(Extracted),
485 /// No JSON at all, with why generation stopped. The reason decides whether
486 /// asking again can possibly help.
487 NoJson {
488 text: String,
489 finish: FinishReason,
490 },
491}
492
493/// How many times a response carrying no JSON at all is asked for again.
494///
495/// Not the same question as the SDK's transport retry, and deliberately a
496/// small number: each attempt is a full reasoning call. The rule this replaced
497/// never retried, justified as "the same prompt truncates the same way" - but
498/// that is [`Extracted::Truncated`], a different branch. A response with *no
499/// JSON at all* did not truncate an answer, it never produced one, and in
500/// practice it does not repeat: drep's own gated push failed on a different
501/// file each run, and each failing file analyzed cleanly when asked again.
502///
503/// Three total attempts, so two local retries before the provider chain may
504/// ask a fallback. Production output has been visibly garbled twice in a row
505/// and then parsed unchanged on a later run; the third attempt salvages that
506/// case without demoting an otherwise healthy provider.
507pub const NO_JSON_ATTEMPTS: u32 = 3;
508
509/// Whether asking the same question again could produce a different answer.
510///
511/// `false` for the reasons that are a property of the *request*: a token cap is
512/// hit identically every time, and a content filter that refused this payload
513/// refuses it again. `true` where the server told us nothing useful, because a
514/// model at temperature above zero can simply answer differently - which is
515/// what drep's own gated push demonstrated, failing on a different file each
516/// run with every failing file analyzing cleanly when asked again.
517fn worth_asking_again(finish: &FinishReason) -> bool {
518 // Written as a negated match on the two request-shaped reasons rather than
519 // as an enumeration of the rest. `FinishReason` is `#[non_exhaustive]`, so
520 // a wildcard arm is required either way - and an enumerated "everything
521 // else is retryable" arm sitting above it is behaviourally identical to the
522 // wildcard, which makes it undeletable-but-unobservable: exactly the dead
523 // code the mutation gate exists to find.
524 //
525 // The consequence of the wildcard is deliberate: a reason a later SDK adds
526 // defaults to retrying. The retry is bounded and cheap to be wrong about,
527 // whereas refusing to retry something transient fails a commit outright.
528 !matches!(
529 finish,
530 // A token cap is hit identically every time - drep sends no
531 // `max_tokens`, so the cap is the server's. A content filter that
532 // refused this payload refuses it again.
533 FinishReason::Length | FinishReason::ContentFilter
534 )
535}
536
537/// A sentence a user can act on, for the reasons that end the attempt.
538///
539/// The two cases want different actions - one is "this file is too big for this
540/// model in one pass", the other is "this provider refused the content" - so
541/// they do not share a message.
542fn stopped_message(finish: &FinishReason, text: &str) -> String {
543 match finish {
544 FinishReason::Length => format!(
545 "the model hit its output token limit before producing any JSON. \
546 This file is too large for this model to review in one request - \
547 split it, or use a provider with a larger output budget. \
548 It managed: {}",
549 excerpt(text, RESPONSE_EXCERPT_MAX)
550 ),
551 _ => format!(
552 "the model stopped ({}) before producing any JSON: {}",
553 finish.as_str(),
554 excerpt(text, RESPONSE_EXCERPT_MAX)
555 ),
556 }
557}
558
559#[cfg(test)]
560mod tests;