Trait LlmClient 

pub trait LlmClient: Send + Sync {
    // Required methods
    fn complete<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        messages: &'life1 [Message],
        system: Option<&'life2 str>,
        tools: &'life3 [ToolDefinition],
    ) -> Pin<Box<dyn Future<Output = Result<LlmResponse>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;
    fn complete_streaming<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        messages: &'life1 [Message],
        system: Option<&'life2 str>,
        tools: &'life3 [ToolDefinition],
        cancel_token: CancellationToken,
    ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;

    // Provided methods
    fn model_generation_concurrency(&self) -> ModelGenerationConcurrency { ... }
    fn model_generation_pool(&self) -> Option<ModelGenerationPool> { ... }
    fn bind_model_generation_admission(
        &self,
        _admission: ModelGenerationAdmission,
        _preadmitted: Option<Arc<ModelGenerationPermit>>,
    ) -> Option<Arc<dyn LlmClient>> { ... }
    fn model_generation_is_managed(&self) -> bool { ... }
    fn take_model_generation_queue_wait(&self) -> Duration { ... }
    fn fork_for_session(&self, _session_id: &str) -> Option<Arc<dyn LlmClient>> { ... }
    fn with_active_generation_timeout(
        &self,
        _timeout: Duration,
    ) -> Option<Arc<dyn LlmClient>> { ... }
    fn native_structured_support(&self) -> NativeStructuredSupport { ... }
    fn has_distinct_non_streaming_transport(&self) -> bool { ... }
    fn complete_structured<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>(
        &'life0 self,
        messages: &'life1 [Message],
        system: Option<&'life2 str>,
        tools: &'life3 [ToolDefinition],
        _directive: &'life4 StructuredDirective,
    ) -> Pin<Box<dyn Future<Output = Result<LlmResponse>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait,
             'life4: 'async_trait { ... }
    fn complete_streaming_structured<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>(
        &'life0 self,
        messages: &'life1 [Message],
        system: Option<&'life2 str>,
        tools: &'life3 [ToolDefinition],
        _directive: &'life4 StructuredDirective,
        cancel_token: CancellationToken,
    ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait,
             'life4: 'async_trait { ... }
}

LLM client trait

Required Methods

fn complete<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, messages: &'life1 [Message], system: Option<&'life2 str>, tools: &'life3 [ToolDefinition], ) -> Pin<Box<dyn Future<Output = Result<LlmResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Complete a conversation (non-streaming)

fn complete_streaming<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, messages: &'life1 [Message], system: Option<&'life2 str>, tools: &'life3 [ToolDefinition], cancel_token: CancellationToken, ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Complete a conversation with streaming Returns a receiver for streaming events. The cancel_token is checked during the HTTP request; if cancelled, the request is aborted.

Provided Methods

fn model_generation_concurrency(&self) -> ModelGenerationConcurrency

Report the client’s explicitly supported active-generation capacity.

The conservative default is single-flight. Providers that can safely serve more active generations must override this with a typed contract; callers must not infer concurrency from provider names or endpoint strings.

fn model_generation_pool(&self) -> Option<ModelGenerationPool>

Describe the non-secret provider/model capacity pool shared by this client. The default is None for custom clients that do not expose a stable routing identity; they retain the existing per-client gate.

fn bind_model_generation_admission( &self, _admission: ModelGenerationAdmission, _preadmitted: Option<Arc<ModelGenerationPermit>>, ) -> Option<Arc<dyn LlmClient>>

Rebind a governed model facade to an invocation-owned generation gate.

This is primarily used when a nested workflow supplies a tighter admission gate than the surrounding session (for example, a Flow step with its own maxConcurrentGenerations limit). Raw provider clients do not need to implement this hook; the agent runtime can wrap them. A client that already owns model-generation admission should preserve the provider transport while replacing the facade’s gate and, when given, consuming the one pre-admitted permit exactly once.

fn model_generation_is_managed(&self) -> bool

Whether this client already applies the model-generation admission contract around every provider call. Built-in run-bound clients use this marker so structured tools do not acquire the same permit twice.

fn take_model_generation_queue_wait(&self) -> Duration

Take queue wait accumulated by a managed client since the previous observation. Unmanaged clients report zero.

fn fork_for_session(&self, _session_id: &str) -> Option<Arc<dyn LlmClient>>

Derive a provider client bound to one logical agent session.

Stateless providers can keep the default and share the existing client. Account-backed providers whose transport uses a live session identity should return an independent client so parallel child agents do not contend for the parent’s active operation.

fn with_active_generation_timeout( &self, _timeout: Duration, ) -> Option<Arc<dyn LlmClient>>

Return a view of this client configured for one active generation deadline. The caller still owns and enforces the outer deadline.

Composite and account-backed clients can use this budget to configure their underlying transport without inferring timeout intent from error text. Stateless clients may keep the default.

fn native_structured_support(&self) -> NativeStructuredSupport

Report the strongest provider-native structured-output enforcement this client supports. Used by structured to decide whether to force a tool call, request a native response_format, or fall back to prompt-and-parse. Defaults to no native support.

fn has_distinct_non_streaming_transport(&self) -> bool

Report whether LlmClient::complete_structured uses a transport that is independent from the streaming implementation.

The conservative default is false because several account-backed clients implement complete by opening a stream and waiting for its terminal event. Composite reliability layers use this capability to avoid presenting the same streaming failure mode as a non-streaming fallback.

fn complete_structured<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>( &'life0 self, messages: &'life1 [Message], system: Option<&'life2 str>, tools: &'life3 [ToolDefinition], _directive: &'life4 StructuredDirective, ) -> Pin<Box<dyn Future<Output = Result<LlmResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait, 'life4: 'async_trait,

Complete a conversation while honoring a structured-output directive (forced tool_choice and/or native response_format).

The default implementation ignores the directive and behaves exactly like LlmClient::complete, so existing clients keep working unchanged; providers that support native structured output override this.

fn complete_streaming_structured<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>( &'life0 self, messages: &'life1 [Message], system: Option<&'life2 str>, tools: &'life3 [ToolDefinition], _directive: &'life4 StructuredDirective, cancel_token: CancellationToken, ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait, 'life4: 'async_trait,

Streaming counterpart of LlmClient::complete_structured. Defaults to LlmClient::complete_streaming, ignoring the directive.

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl LlmClient for AnthropicClient

impl LlmClient for CodexLoginClient

impl LlmClient for OpenAiClient

impl LlmClient for ZhipuClient