Skip to main content

switchyard_llm_client/
observation.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Request-scoped observations emitted while serving an algorithm run.
5
6use std::sync::Arc;
7use std::time::Duration;
8
9use switchyard_libsy::OutcomeMetadata;
10use switchyard_protocol::{ModelId, Usage};
11
12/// One completed model call observed while serving an algorithm run.
13#[derive(Clone, Debug)]
14pub struct LlmCallObservation {
15    /// Model selected for the completed call.
16    pub selected_model: ModelId,
17    /// Whether the call completed successfully.
18    pub is_success: bool,
19    /// Time spent waiting for the model call to resolve.
20    pub duration: Duration,
21    /// Normalized usage for a buffered successful response.
22    pub usage: Option<Usage>,
23}
24
25/// Events emitted inline while [`crate::run`] serves a routing request.
26#[derive(Clone, Debug)]
27pub enum RunObservation {
28    /// Metadata attached to the completed routing outcome.
29    Outcome(OutcomeMetadata),
30    /// A completed model call requested by the algorithm for routing work.
31    LlmCall(LlmCallObservation),
32    /// A completed terminal model call made from the routing outcome.
33    AnswerCall(LlmCallObservation),
34    /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
35    RoutingOverhead(Duration),
36}
37
38/// Request-scoped callback for algorithm-run observations.
39///
40/// The runner invokes this callback inline while resolving observations. Several
41/// runs may invoke the same observer concurrently, so implementations must be
42/// thread-safe, fast, and non-blocking.
43pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;