objectiveai_api/ctx/
ctx.rs

1//! Request context containing per-request state and caches.
2
3use dashmap::DashMap;
4use futures::future::Shared;
5use std::sync::Arc;
6
7/// Per-request context containing user-specific state and deduplication caches.
8///
9/// The context is generic over `CTXEXT`, allowing custom extensions for
10/// different deployment scenarios (e.g., different BYOK providers).
11///
12/// # Caches
13///
14/// The caches deduplicate concurrent fetches for the same resource within a request.
15/// When multiple parts of a request need the same ensemble or ensemble LLM,
16/// only one fetch is performed and the result is shared.
17#[derive(Debug)]
18pub struct Context<CTXEXT> {
19    /// Custom context extension (e.g., for BYOK keys).
20    pub ext: Arc<CTXEXT>,
21    /// Multiplier applied to costs for this request.
22    pub cost_multiplier: rust_decimal::Decimal,
23    /// Cache for ensemble fetches, keyed by ensemble ID.
24    pub ensemble_cache: Arc<
25        DashMap<
26            String,
27            Shared<
28                tokio::sync::oneshot::Receiver<
29                    Result<
30                        Option<(objectiveai::ensemble::Ensemble, u64)>,
31                        objectiveai::error::ResponseError,
32                    >,
33                >,
34            >,
35        >,
36    >,
37    /// Cache for ensemble LLM fetches, keyed by ensemble LLM ID.
38    pub ensemble_llm_cache: Arc<
39        DashMap<
40            String,
41            Shared<
42                tokio::sync::oneshot::Receiver<
43                    Result<
44                        Option<(objectiveai::ensemble_llm::EnsembleLlm, u64)>,
45                        objectiveai::error::ResponseError,
46                    >,
47                >,
48            >,
49        >,
50    >,
51}
52
53impl<CTXEXT> Clone for Context<CTXEXT> {
54    fn clone(&self) -> Self {
55        Self {
56            ext: self.ext.clone(),
57            cost_multiplier: self.cost_multiplier,
58            ensemble_cache: self.ensemble_cache.clone(),
59            ensemble_llm_cache: self.ensemble_llm_cache.clone(),
60        }
61    }
62}
63
64impl<CTXEXT> Context<CTXEXT> {
65    /// Creates a new context with the given extension and cost multiplier.
66    pub fn new(ext: Arc<CTXEXT>, cost_multiplier: rust_decimal::Decimal) -> Self {
67        Self {
68            ext,
69            cost_multiplier,
70            ensemble_cache: Arc::new(DashMap::new()),
71            ensemble_llm_cache: Arc::new(DashMap::new()),
72        }
73    }
74}
objectiveai_api/ctx/ctx.rs

objectiveai_api/ctx/
ctx.rs
