Struct LlmCachePolicy 

pub struct LlmCachePolicy {
    pub key_func: Option<Arc<dyn Fn(&LlmCacheKeyInput) -> String + Send + Sync>>,
}

Cache policy for LLM response caching

Controls how LLM responses are cached, including the ability to customize the cache key generation function.

Examples

use juncture_tracing::types::LlmCachePolicy;

let policy = LlmCachePolicy::default();
assert!(policy.key_func.is_none());

Fields

key_func: Option<Arc<dyn Fn(&LlmCacheKeyInput) -> String + Send + Sync>>

Optional custom cache key function

If None, the default key function using (model, messages_hash) is used.

Implementations

impl LlmCachePolicy

pub fn new() -> Self

Create a new cache policy with the default key function

Examples

use juncture_tracing::types::LlmCachePolicy;

let policy = LlmCachePolicy::new();
assert!(policy.key_func.is_none());

pub fn with_key_func<F>(self, f: F) -> Self

where F: Fn(&LlmCacheKeyInput) -> String + Send + Sync + 'static,

Set a custom cache key function

Parameters

• f - Function that generates cache keys from input

Examples

use juncture_tracing::types::{LlmCachePolicy, LlmCacheKeyInput};
use serde_json::json;

let policy = LlmCachePolicy::new().with_key_func(|input| {
    format!("{}:{}", input.model, input.messages.len())
});

let key_input = LlmCacheKeyInput {
    model: "gpt-4".to_string(),
    messages: vec![],
    tools: vec![],
    config: None,
};
assert!(policy.key_func.as_ref().map(|f| f(&key_input)).is_some());

Trait Implementations

impl Debug for LlmCachePolicy

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlmCachePolicy

fn default() -> LlmCachePolicy

Returns the “default value” for a type.