klieo-core 3.13.0

//! Conformance properties for klieo's memory traits.
//!
//! klieo ships one trait per memory concern and many impls per trait. These
//! properties are what every impl must agree on, so a divergence between a
//! fake and a real backend becomes a failing test rather than a surprise in
//! production. Run them from a backend's own test file:
//!
//! ```ignore
//! #[tokio::test]
//! async fn satisfies_long_term_conformance() {
//!     let store = MyLongTerm::new();
//!     let scopes = Scopes::agent("conformance-primary", "conformance-other");
//!     conformance::long_term_memory(&store, &scopes, ExpectedOrdering::Relevance {
//!         nearer: "the sky is blue".into(),
//!         farther: "diesel engine maintenance".into(),
//!         query: "what colour is the sky".into(),
//!     })
//!     .await;
//! }
//! ```
//!
//! Conformance is opt-in: an impl that deliberately does nothing (an
//! intentional no-op store) simply never calls these, and says why in its own
//! docs.
//!
//! Every property here holds regardless of whether recall is a substring
//! match, a cosine-similarity search, or a database predicate. That is why
//! none of them assert "a non-matching query returns nothing" — a pure vector
//! backend with no score threshold returns its top `k` however unrelated the
//! query, and asserting otherwise would fail a correct implementation.
//!
//! Each property uses its own marker token, so running them against one store
//! in sequence cannot let one property's facts pollute another's assertions.

use crate::ids::{FactId, ThreadId};
use crate::llm::{Message, Role};
use crate::memory::{Fact, LongTermMemory, Scope, ShortTermMemory};

/// The two scopes a long-term conformance run needs.
///
/// Supplied by the caller rather than invented here: a shared Qdrant
/// collection or a reused table is the normal case, and avoiding collisions
/// with other tests belongs to whoever knows the deployment.
pub struct Scopes {
    /// Scope the properties store into and recall from.
    pub primary: Scope,
    /// A different scope, which must never see `primary`'s facts.
    pub other: Scope,
}

impl Scopes {
    /// Build both scopes as [`Scope::Agent`] from two distinct names.
    pub fn agent(primary: impl Into<String>, other: impl Into<String>) -> Self {
        Self {
            primary: Scope::Agent(primary.into()),
            other: Scope::Agent(other.into()),
        }
    }
}

/// The recall ordering an impl claims to provide.
///
/// Passed in rather than read off the trait: the only consumer of an ordering
/// declaration is this suite, so widening [`LongTermMemory`] to carry one
/// would be a frozen-trait-surface change for a test-only concern.
pub enum ExpectedOrdering {
    /// Most recently stored first. This suite owns the fixture.
    Recency,
    /// Most semantically relevant first.
    ///
    /// The caller supplies the pair because only they know what "nearer"
    /// means for their embedder. This is the one place an implementation can
    /// supply a fixture that passes trivially — accepted, because vector
    /// ranking is not the divergence this suite exists to catch.
    Relevance {
        /// Text that must rank above `farther` for `query`.
        nearer: String,
        /// Text that must rank below `nearer` for `query`.
        farther: String,
        /// Query both texts are stored against.
        query: String,
    },
    /// No ordering guarantee. The assertion is skipped, but stating this is
    /// deliberate — callers of such a store must not depend on order.
    Unspecified,
}

const ORDERING_TOKEN: &str = "klieo-conformance-ordering";
const ISOLATION_TOKEN: &str = "klieo-conformance-isolation";
const K_BOUND_TOKEN: &str = "klieo-conformance-kbound";
const FORGET_TOKEN: &str = "klieo-conformance-forget";
const K_BOUND_STORED: usize = 3;
const K_BOUND_REQUESTED: usize = 2;
const RECALL_K: usize = 10;

/// Run every shipped [`LongTermMemory`] property, in order.
///
/// The empty-store property runs first because it doubles as the precondition
/// check: these properties require a store with nothing in the given scopes.
pub async fn long_term_memory(
    store: &dyn LongTermMemory,
    scopes: &Scopes,
    ordering: ExpectedOrdering,
) {
    long_term_empty_store_recalls_nothing(store, scopes).await;
    long_term_scopes_are_isolated(store, scopes).await;
    long_term_recall_returns_at_most_k(store, scopes).await;
    long_term_forget_removes_the_fact(store, scopes).await;
    long_term_recall_ordering_matches_declaration(store, scopes, ordering).await;
}

/// A store with nothing in scope returns an empty vec, never an error.
pub async fn long_term_empty_store_recalls_nothing(store: &dyn LongTermMemory, scopes: &Scopes) {
    for scope in [&scopes.primary, &scopes.other] {
        let found = recall(store, scope, "klieo-conformance-precondition-probe").await;
        assert!(
            found.is_empty(),
            "precondition failed: scope must be empty at entry, found {} fact(s)",
            found.len()
        );
    }
}

/// A fact stored under one scope is never returned for another.
pub async fn long_term_scopes_are_isolated(store: &dyn LongTermMemory, scopes: &Scopes) {
    let text = format!("{ISOLATION_TOKEN} probe");
    remember(store, &scopes.primary, &text).await;

    let in_primary = recall(store, &scopes.primary, ISOLATION_TOKEN).await;
    assert!(
        contains_text(&in_primary, &text),
        "a fact stored under the primary scope was not recalled from it"
    );

    let in_other = recall(store, &scopes.other, ISOLATION_TOKEN).await;
    assert!(
        !contains_text(&in_other, &text),
        "a fact stored under one scope leaked into another"
    );
}

/// Recall never returns more than the requested `k`.
pub async fn long_term_recall_returns_at_most_k(store: &dyn LongTermMemory, scopes: &Scopes) {
    for i in 0..K_BOUND_STORED {
        remember(
            store,
            &scopes.primary,
            &format!("{K_BOUND_TOKEN} entry {i}"),
        )
        .await;
    }
    let found = store
        .recall(scopes.primary.clone(), K_BOUND_TOKEN, K_BOUND_REQUESTED)
        .await
        .expect("recall must not error");
    assert!(
        found.len() <= K_BOUND_REQUESTED,
        "recall returned {} facts for k={K_BOUND_REQUESTED}",
        found.len()
    );
}

/// A forgotten fact is never recalled again, and forgetting an unknown id is
/// not an error.
pub async fn long_term_forget_removes_the_fact(store: &dyn LongTermMemory, scopes: &Scopes) {
    let text = format!("{FORGET_TOKEN} probe");
    let id = remember(store, &scopes.primary, &text).await;
    assert!(
        contains_text(&recall(store, &scopes.primary, FORGET_TOKEN).await, &text),
        "a stored fact was not recalled before forgetting it"
    );

    store
        .forget(id.clone())
        .await
        .expect("forget must not error");
    assert!(
        !contains_text(&recall(store, &scopes.primary, FORGET_TOKEN).await, &text),
        "a forgotten fact was still recalled"
    );

    // Forgetting an id that is no longer present must be a no-op, not an error.
    //
    // Reuses the id this store itself minted rather than inventing one. An
    // earlier draft passed a literal `"klieo-conformance-never-stored"`, which
    // conflates "absent" with "malformed": `PgvectorLongTerm::forget` parses
    // the id as a UUID and deliberately rejects anything else (it has a test
    // named `parse_fact_id_rejects_non_uuid`). That would have failed a correct
    // backend for validating its input. The property is about absence, so the
    // id has to be one the store would accept.
    store
        .forget(id)
        .await
        .expect("forgetting an id that is no longer present must not error");
}

/// Recall order matches what the impl declares.
pub async fn long_term_recall_ordering_matches_declaration(
    store: &dyn LongTermMemory,
    scopes: &Scopes,
    ordering: ExpectedOrdering,
) {
    match ordering {
        ExpectedOrdering::Unspecified => {}
        ExpectedOrdering::Recency => {
            let earlier = format!("{ORDERING_TOKEN} earlier");
            let later = format!("{ORDERING_TOKEN} later");
            remember(store, &scopes.primary, &earlier).await;
            remember(store, &scopes.primary, &later).await;
            assert_ranks_above(
                &recall(store, &scopes.primary, ORDERING_TOKEN).await,
                &later,
                &earlier,
                "declared Recency, but the older fact ranked first",
            );
        }
        ExpectedOrdering::Relevance {
            nearer,
            farther,
            query,
        } => {
            remember(store, &scopes.primary, &farther).await;
            remember(store, &scopes.primary, &nearer).await;
            assert_ranks_above(
                &recall(store, &scopes.primary, &query).await,
                &nearer,
                &farther,
                "declared Relevance, but the less relevant fact ranked first",
            );
        }
    }
}

async fn remember(store: &dyn LongTermMemory, scope: &Scope, text: &str) -> FactId {
    store
        .remember(scope.clone(), Fact::new(text))
        .await
        .expect("remember must not error")
}

async fn recall(store: &dyn LongTermMemory, scope: &Scope, query: &str) -> Vec<Fact> {
    store
        .recall(scope.clone(), query, RECALL_K)
        .await
        .expect("recall must not error")
}

fn contains_text(facts: &[Fact], text: &str) -> bool {
    facts.iter().any(|f| f.text == text)
}

fn position_of(facts: &[Fact], text: &str) -> Option<usize> {
    facts.iter().position(|f| f.text == text)
}

/// Asserts `above` outranks `below`.
///
/// Tolerates other facts between them, so one property's fixtures cannot break
/// another's assertion — and tolerates `below` being absent entirely, which is
/// what a narrowing or thresholding store legitimately does to a less relevant
/// fact. Excluding it *is* ranking it last; demanding its presence would fail a
/// correct implementation.
///
/// `above` must be present. That is the real assertion, and for a narrowing
/// store it is the meaningful one: it proves the narrowing kept the right fact.
///
/// Learned from `GraphAwareLongTerm`, which narrows candidates by graph entity
/// before the vector search. An earlier draft required both fixture facts back
/// and failed it for behaving as designed.
fn assert_ranks_above(facts: &[Fact], above: &str, below: &str, message: &str) {
    let above_at = position_of(facts, above)
        .unwrap_or_else(|| panic!("{message}: the expected-first fact was not recalled at all"));
    if let Some(below_at) = position_of(facts, below) {
        assert!(above_at < below_at, "{message}");
    }
}

const MESSAGE_COUNT: usize = 10;
const MESSAGE_BODY_CHARS: usize = 200;
const GENEROUS_TOKEN_BUDGET: usize = 8_000;
const TIGHT_TOKEN_BUDGET: usize = 100;

/// Run every shipped [`ShortTermMemory`] property.
pub async fn short_term_memory(store: &dyn ShortTermMemory) {
    short_term_load_honours_max_tokens(store).await;
    short_term_budget_is_encoding_independent(store).await;
}

const ASCII_CELL: &str = "aaaa";
const MULTIBYTE_CELL: &str = "日本語だ";
const ENCODING_PROBE_MESSAGES: usize = 6;
const ENCODING_PROBE_BUDGET: usize = 6;

/// The budget measures text, not bytes: two threads whose messages have the
/// same *character* count must keep the same number of messages.
///
/// Deliberately prescribes no constant — an implementation may approximate
/// tokens however it likes, per [`ShortTermMemory::load`]'s contract. What it
/// may not do is charge the same amount of text three times more because it
/// happens to be CJK, emoji, or accented.
///
/// Added when `SqliteShortTerm` and `InMemoryShortTerm`, both passing every
/// other property here, were measured returning 1 vs 3 and 2 vs 6 messages for
/// identical multibyte input: sqlite charged `content.len() / 4` (UTF-8 bytes)
/// while the shared helper charged `chars().count()`. Nothing caught it,
/// because monotonicity alone cannot.
pub async fn short_term_budget_is_encoding_independent(store: &dyn ShortTermMemory) {
    let ascii = ThreadId::new("klieo-conformance-encoding-ascii");
    let multibyte = ThreadId::new("klieo-conformance-encoding-multibyte");
    assert_eq!(
        ASCII_CELL.chars().count(),
        MULTIBYTE_CELL.chars().count(),
        "fixture bug: the two probe strings must be the same character length"
    );

    for _ in 0..ENCODING_PROBE_MESSAGES {
        for (thread, cell) in [(&ascii, ASCII_CELL), (&multibyte, MULTIBYTE_CELL)] {
            store
                .append(thread.clone(), text_message(cell))
                .await
                .expect("append must not error");
        }
    }

    let ascii_kept = load(store, &ascii, ENCODING_PROBE_BUDGET).await.len();
    let multibyte_kept = load(store, &multibyte, ENCODING_PROBE_BUDGET).await.len();

    for thread in [ascii, multibyte] {
        store.clear(thread).await.expect("clear must not error");
    }

    assert_eq!(
        ascii_kept, multibyte_kept,
        "the same amount of text was charged differently by encoding: kept \
         {ascii_kept} ASCII messages but {multibyte_kept} multibyte ones at a \
         budget of {ENCODING_PROBE_BUDGET}. Count characters, not UTF-8 bytes"
    );
}

fn text_message(content: &str) -> Message {
    Message {
        role: Role::User,
        content: content.to_string(),
        tool_calls: vec![],
        tool_call_id: None,
    }
}

/// A tighter `max_tokens` budget returns strictly fewer messages.
///
/// This is the one mechanism bounding how much history reaches a model, and
/// an impl that ignores the argument silently removes that bound for every
/// caller.
pub async fn short_term_load_honours_max_tokens(store: &dyn ShortTermMemory) {
    let thread = ThreadId::new("klieo-conformance-max-tokens");
    for i in 0..MESSAGE_COUNT {
        store
            .append(thread.clone(), padded_message(i))
            .await
            .expect("append must not error");
    }

    let generous = load(store, &thread, GENEROUS_TOKEN_BUDGET).await;
    assert_eq!(
        generous.len(),
        MESSAGE_COUNT,
        "a generous budget must return the whole history"
    );

    let tight = load(store, &thread, TIGHT_TOKEN_BUDGET).await;
    assert!(
        tight.len() < generous.len(),
        "load ignored max_tokens: {} messages returned for a budget of {TIGHT_TOKEN_BUDGET} \
         tokens, same as for {GENEROUS_TOKEN_BUDGET}",
        tight.len()
    );

    store.clear(thread).await.expect("clear must not error");
}

async fn load(store: &dyn ShortTermMemory, thread: &ThreadId, max_tokens: usize) -> Vec<Message> {
    store
        .load(thread.clone(), max_tokens)
        .await
        .expect("load must not error")
}

fn padded_message(index: usize) -> Message {
    Message {
        role: Role::User,
        content: format!("message {index} {}", "x".repeat(MESSAGE_BODY_CHARS)),
        tool_calls: vec![],
        tool_call_id: None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::error::MemoryError;
    use crate::test_utils::{InMemoryLongTerm, InMemoryShortTerm};
    use async_trait::async_trait;
    use tokio::sync::Mutex;

    fn scopes() -> Scopes {
        Scopes::agent("conformance-primary", "conformance-other")
    }

    #[tokio::test]
    async fn in_memory_long_term_satisfies_conformance() {
        long_term_memory(
            &InMemoryLongTerm::default(),
            &scopes(),
            ExpectedOrdering::Recency,
        )
        .await;
    }

    #[tokio::test]
    async fn in_memory_short_term_satisfies_conformance() {
        short_term_memory(&InMemoryShortTerm::default()).await;
    }

    /// A store returning oldest-first — `InMemoryLongTerm`'s behaviour before
    /// this suite existed. Proves the ordering property is not vacuous.
    #[derive(Default)]
    struct OldestFirstLongTerm {
        facts: Mutex<Vec<(FactId, Scope, Fact)>>,
    }

    #[async_trait]
    impl LongTermMemory for OldestFirstLongTerm {
        async fn remember(&self, scope: Scope, fact: Fact) -> Result<FactId, MemoryError> {
            let mut facts = self.facts.lock().await;
            let id = FactId::new(format!("oldest-first-{}", facts.len()));
            facts.push((id.clone(), scope, fact));
            Ok(id)
        }

        async fn recall(
            &self,
            scope: Scope,
            query: &str,
            k: usize,
        ) -> Result<Vec<Fact>, MemoryError> {
            let q = query.to_lowercase();
            Ok(self
                .facts
                .lock()
                .await
                .iter()
                .filter(|(_, s, _)| *s == scope)
                .filter(|(_, _, f)| f.text.to_lowercase().contains(&q))
                .take(k)
                .map(|(_, _, f)| f.clone())
                .collect())
        }

        async fn forget(&self, id: FactId) -> Result<(), MemoryError> {
            self.facts.lock().await.retain(|(i, _, _)| i != &id);
            Ok(())
        }
    }

    #[tokio::test]
    #[should_panic(expected = "declared Recency")]
    async fn ordering_property_rejects_a_store_that_returns_oldest_first() {
        long_term_recall_ordering_matches_declaration(
            &OldestFirstLongTerm::default(),
            &scopes(),
            ExpectedOrdering::Recency,
        )
        .await;
    }

    /// A store ignoring `max_tokens` — `InMemoryShortTerm`'s behaviour before
    /// this suite existed. Proves the budget property is not vacuous.
    #[derive(Default)]
    struct UnboundedShortTerm {
        messages: Mutex<Vec<Message>>,
    }

    #[async_trait]
    impl ShortTermMemory for UnboundedShortTerm {
        async fn append(&self, _thread: ThreadId, msg: Message) -> Result<(), MemoryError> {
            self.messages.lock().await.push(msg);
            Ok(())
        }

        async fn load(
            &self,
            _thread: ThreadId,
            _max_tokens: usize,
        ) -> Result<Vec<Message>, MemoryError> {
            Ok(self.messages.lock().await.clone())
        }

        async fn clear(&self, _thread: ThreadId) -> Result<(), MemoryError> {
            self.messages.lock().await.clear();
            Ok(())
        }
    }

    #[tokio::test]
    #[should_panic(expected = "load ignored max_tokens")]
    async fn budget_property_rejects_a_store_that_ignores_max_tokens() {
        short_term_load_honours_max_tokens(&UnboundedShortTerm::default()).await;
    }
}

#[cfg(test)]
mod backlog_regression_tests {
    //! Guards for the small fixes in backlog items 5, 6 and 8 — each asserts
    //! the thing an adopter would actually hit, not the mechanism.

    /// Item 8: the error a module's traits raise must be reachable from that
    /// module. These `use`s failing to compile IS the regression.
    #[allow(unused_imports)]
    mod reachable_paths {
        use crate::llm::LlmError;
        use crate::tool::ToolError;
    }

    /// Item 6: a fake must be able to fail on demand with a chosen variant,
    /// rather than only via script exhaustion (which yields a fixed
    /// `BadRequest` regardless of what the test is about).
    #[tokio::test]
    async fn fake_llm_step_error_returns_the_chosen_variant() {
        use crate::error::LlmError;
        use crate::llm::{ChatRequest, LlmClient};
        use crate::test_utils::{FakeLlmClient, FakeLlmStep};

        let llm =
            FakeLlmClient::new("fake").with_steps(vec![FakeLlmStep::Error(LlmError::RateLimit {
                retry_after_secs: 42,
            })]);

        match llm.complete(ChatRequest::new(vec![])).await {
            Err(LlmError::RateLimit {
                retry_after_secs: 42,
            }) => {}
            other => panic!("expected the scripted RateLimit, got {other:?}"),
        }
    }
}