mnem_bench/
adapter.rs

1//! `BenchAdapter` trait - what every system-under-test (mnem, mem0,
2//! MemPalace) implements so the scorers can drive it.
3
4use serde::{Deserialize, Serialize};
5use std::error::Error as StdError;
6
7/// One document staged for ingest. The scorer assigns the
8/// `external_id` (e.g. session id, dialog id) and reads it back from
9/// the retrieved hits via [`Hit::external_id`] so the adapter is
10/// free to mint any internal id it likes.
11#[derive(Clone, Debug, Serialize, Deserialize)]
12pub struct IngestDoc {
13    /// Stable per-bench identifier the scorer attaches to this doc.
14    /// Echoed back unchanged on the matching [`Hit`].
15    pub external_id: String,
16    /// Scope label used to keep the corpus per-question / per-conv
17    /// isolated. mnem encodes this as the `Node.ntype` so the
18    /// retriever's label filter scopes to a single question's
19    /// haystack.
20    pub label: String,
21    /// Free-form natural-language text the embedder consumes.
22    pub text: String,
23    /// Optional structured property bag. Echoed onto the node /
24    /// document for downstream filtering. Values are stringified by
25    /// adapters that cannot store arbitrary JSON.
26    pub props: serde_json::Map<String, serde_json::Value>,
27}
28
29/// One hit from a retrieve. `external_id` matches the value the
30/// scorer staged on the corresponding [`IngestDoc`]. Adapters that
31/// re-rank or filter still preserve `external_id` round-trip.
32#[derive(Clone, Debug, Serialize, Deserialize)]
33pub struct Hit {
34    /// External id originally staged with [`IngestDoc::external_id`].
35    pub external_id: String,
36    /// Adapter-internal ranking score. Higher is better. The scorer
37    /// only consumes the rank order, but the score is logged for
38    /// debugging.
39    pub score: f32,
40}
41
42/// Adapter trait. One instance handles a single benchmark run.
43pub trait BenchAdapter {
44    /// Drop all per-question / per-conv state so the next ingest is
45    /// fresh. mnem's in-memory adapter rotates a new `Repo`; HTTP
46    /// adapters POST a label-scoped delete.
47    ///
48    /// # Errors
49    ///
50    /// Returns the adapter's own error type when the reset fails.
51    fn reset(&mut self) -> Result<(), Box<dyn StdError>>;
52
53    /// Ingest the given documents under their staged labels. Caller
54    /// is responsible for [`Self::reset`] between unrelated batches.
55    ///
56    /// # Errors
57    ///
58    /// Returns adapter-specific errors when the underlying store
59    /// rejects a document.
60    fn ingest(&mut self, docs: &[IngestDoc]) -> Result<(), Box<dyn StdError>>;
61
62    /// Retrieve the top-K matches for `query` within `label`.
63    /// Returned hits MUST be ordered score-desc. The scorer only
64    /// looks at the first `top_k` entries.
65    ///
66    /// # Errors
67    ///
68    /// Returns adapter-specific errors when the underlying store
69    /// rejects the query.
70    fn retrieve(
71        &mut self,
72        label: &str,
73        query: &str,
74        top_k: usize,
75    ) -> Result<Vec<Hit>, Box<dyn StdError>>;
76
77    /// Free-form adapter name for logs + RESULTS.md. e.g. `"mnem"`.
78    fn name(&self) -> &str;
79}
mnem_bench/adapter.rs

mnem_bench/
adapter.rs
