Skip to main content

ocp_conformance/
lib.rs

1//! `ocp-conformance` — the public Open Context Protocol conformance suite
2//! (`06-context-protocol.md` §3.6).
3//!
4//! "OCP conformant" means *green on this suite for your declared capability
5//! set* — a checkable claim, which is what makes third-party adoption safe.
6//! [`run_conformance`] drives a provider through the protocol and returns a
7//! typed [`ConformanceReport`] with a pass/fail verdict per check and an
8//! evidence string for each, so a failure says exactly what was wrong.
9//!
10//! The checks (all against the frozen `ocp-types` contracts):
11//!
12//! - **handshake** — the provider completes the handshake and reports a
13//!   non-empty identity + capabilities (§3.2).
14//! - **frame-validity** — queried frames pass `ocp-types` validation: score
15//!   in `[0, 1]`, a non-empty title, a non-empty `citation_label` (§3.4 —
16//!   "NEVER a bare uuid").
17//! - **budget-honesty** — returned frames' summed `token_cost` never exceeds
18//!   the query budget (§3.3 — "never lies about cost").
19//! - **shutdown-clean** — the provider tears down without error (§3.2).
20//! - **malformed-input-tolerance** — a garbage line is ignored-or-errored,
21//!   never crashing the host (§3.5, task deliverable). Wire-level, so it
22//!   applies to stdio providers.
23//!
24//! The suite is deliberately adversarial: pointed at a provider that lies
25//! about costs, emits an out-of-range score, omits a citation label, or dies
26//! mid-query, the matching check fails loudly. The bundled `ocp-example-docs`
27//! fixture has `--misbehave` flags that trip each one, proving the suite
28//! catches a broken provider (task deliverable).
29
30use ocp_host::{ConsentRecord, ContextProvider, Host, HostError, RawStdioConnection};
31use ocp_types::{Capabilities, ContextQuery, ContextQueryResult, ProviderInfo};
32
33mod report;
34
35pub use report::{CheckResult, CheckStatus, ConformanceReport};
36
37/// The stable check names, so reports and callers agree on identifiers.
38pub const CHECK_HANDSHAKE: &str = "handshake";
39pub const CHECK_FRAME_VALIDITY: &str = "frame-validity";
40pub const CHECK_BUDGET_HONESTY: &str = "budget-honesty";
41pub const CHECK_SHUTDOWN: &str = "shutdown-clean";
42pub const CHECK_MALFORMED: &str = "malformed-input-tolerance";
43
44/// How to reach the provider under test. `ocp-inspect` builds one of these
45/// from its CLI arguments; tests build them directly.
46pub enum ProviderTarget {
47    /// A child-process provider: `program` plus `args`.
48    Stdio { program: String, args: Vec<String> },
49    /// A remote provider at `url`.
50    Http { url: String },
51    /// An already-constructed in-process provider (e.g. a built-in).
52    InProcess(Box<dyn ContextProvider>),
53}
54
55impl ProviderTarget {
56    /// A one-line human description of the target, for the report header.
57    pub fn describe(&self) -> String {
58        match self {
59            ProviderTarget::Stdio { program, args } => {
60                if args.is_empty() {
61                    format!("stdio: {program}")
62                } else {
63                    format!("stdio: {program} {}", args.join(" "))
64                }
65            }
66            ProviderTarget::Http { url } => format!("http: {url}"),
67            ProviderTarget::InProcess(provider) => format!("in-process: {}", provider.id()),
68        }
69    }
70}
71
72/// Run the full conformance suite against a provider, returning a typed
73/// report. Never panics: every failure mode becomes a failing check with
74/// evidence.
75pub async fn run_conformance(target: ProviderTarget) -> ConformanceReport {
76    let description = target.describe();
77
78    // Capture stdio spawn info before `target` is consumed — the malformed
79    // probe needs a second, independent connection to the same program.
80    let stdio_probe = match &target {
81        ProviderTarget::Stdio { program, args } => Some((program.clone(), args.clone())),
82        _ => None,
83    };
84
85    let mut checks = Vec::new();
86
87    match build_host(target).await {
88        Ok((host, id, info, caps)) => {
89            if info.name.trim().is_empty() || info.version.trim().is_empty() {
90                checks.push(CheckResult::fail(
91                    CHECK_HANDSHAKE,
92                    format!(
93                        "provider identity incomplete: name='{}' version='{}'",
94                        info.name, info.version
95                    ),
96                ));
97            } else {
98                checks.push(CheckResult::pass(
99                    CHECK_HANDSHAKE,
100                    describe_handshake(&info, &caps),
101                ));
102            }
103            run_query_and_shutdown_checks(host, &id, &mut checks).await;
104        }
105        Err(error) => {
106            checks.push(CheckResult::fail(
107                CHECK_HANDSHAKE,
108                format!("could not establish provider: {error}"),
109            ));
110            for name in [CHECK_FRAME_VALIDITY, CHECK_BUDGET_HONESTY, CHECK_SHUTDOWN] {
111                checks.push(CheckResult::skip(name, "handshake failed"));
112            }
113        }
114    }
115
116    match stdio_probe {
117        Some((program, args)) => checks.push(malformed_stdio_probe(&program, &args).await),
118        None => checks.push(CheckResult::skip(
119            CHECK_MALFORMED,
120            "wire-level malformed-input probe applies to stdio providers only",
121        )),
122    }
123
124    ConformanceReport {
125        target: description,
126        checks,
127    }
128}
129
130/// Stand up a one-provider host for the target and read back the provider's
131/// negotiated identity + capabilities. Records consent for an egress
132/// provider under test — running the suite *is* the consent to its declared
133/// flow, so it isn't spuriously gated.
134async fn build_host(
135    target: ProviderTarget,
136) -> Result<(Host, String, ProviderInfo, Capabilities), HostError> {
137    let mut host = Host::new();
138    let (id, info, caps) = match target {
139        ProviderTarget::Stdio { program, args } => {
140            let id = "provider-under-test".to_string();
141            host.add_stdio(id.clone(), &program, &args).await?;
142            capture_identity(&host, &id)?
143        }
144        ProviderTarget::Http { url } => {
145            let id = "provider-under-test".to_string();
146            host.add_http(id.clone(), url).await?;
147            capture_identity(&host, &id)?
148        }
149        ProviderTarget::InProcess(provider) => {
150            let id = provider.id().to_string();
151            let info = provider.info().clone();
152            let caps = provider.capabilities().clone();
153            host.register(provider);
154            (id, info, caps)
155        }
156    };
157
158    if info.data_flow.egress {
159        host.record_consent(ConsentRecord::new(
160            id.clone(),
161            info.data_flow,
162            "conformance run under test",
163        ));
164    }
165
166    Ok((host, id, info, caps))
167}
168
169fn capture_identity(
170    host: &Host,
171    id: &str,
172) -> Result<(String, ProviderInfo, Capabilities), HostError> {
173    let provider = host
174        .provider(id)
175        .ok_or_else(|| HostError::UnknownProvider(id.to_string()))?;
176    Ok((
177        id.to_string(),
178        provider.info().clone(),
179        provider.capabilities().clone(),
180    ))
181}
182
183async fn run_query_and_shutdown_checks(host: Host, id: &str, checks: &mut Vec<CheckResult>) {
184    let query = sample_query();
185    match host.query_provider(id, &query).await {
186        Ok(result) => {
187            let (ok, evidence) = check_frames(&result);
188            checks.push(CheckResult::from_bool(CHECK_FRAME_VALIDITY, ok, evidence));
189
190            if result.respects_budget(query.max_tokens) {
191                checks.push(CheckResult::pass(
192                    CHECK_BUDGET_HONESTY,
193                    format!(
194                        "{} frame(s) sum to {} token(s), within the {}-token budget",
195                        result.frames.len(),
196                        result.total_token_cost(),
197                        query.max_tokens
198                    ),
199                ));
200            } else {
201                checks.push(CheckResult::fail(
202                    CHECK_BUDGET_HONESTY,
203                    format!(
204                        "frames claim {} token(s), over the {}-token budget — token_cost is dishonest",
205                        result.total_token_cost(),
206                        query.max_tokens
207                    ),
208                ));
209            }
210        }
211        Err(error) => {
212            let evidence = format!("query failed: {error}");
213            checks.push(CheckResult::fail(CHECK_FRAME_VALIDITY, evidence.clone()));
214            checks.push(CheckResult::fail(CHECK_BUDGET_HONESTY, evidence));
215        }
216    }
217
218    let results = host.shutdown().await;
219    match results.iter().find(|(pid, _)| pid == id) {
220        Some((_, Ok(()))) => checks.push(CheckResult::pass(
221            CHECK_SHUTDOWN,
222            "provider acknowledged shutdown and tore down cleanly",
223        )),
224        Some((_, Err(error))) => checks.push(CheckResult::fail(
225            CHECK_SHUTDOWN,
226            format!("shutdown error: {error}"),
227        )),
228        None => checks.push(CheckResult::fail(
229            CHECK_SHUTDOWN,
230            "provider vanished before shutdown could be attempted",
231        )),
232    }
233}
234
235/// Wire-level probe: complete the handshake on a fresh connection, inject a
236/// malformed line, then send a valid query. A conforming provider ignores or
237/// cleanly errors on the garbage and stays alive to answer the query; a
238/// provider that dies on one bad line fails (§3.5).
239async fn malformed_stdio_probe(program: &str, args: &[String]) -> CheckResult {
240    let mut conn = match RawStdioConnection::spawn(program, args).await {
241        Ok(conn) => conn,
242        Err(error) => {
243            return CheckResult::fail(
244                CHECK_MALFORMED,
245                format!("could not spawn provider: {error}"),
246            );
247        }
248    };
249    if let Err(error) = conn.handshake().await {
250        return CheckResult::fail(
251            CHECK_MALFORMED,
252            format!("handshake failed before the probe could run: {error}"),
253        );
254    }
255    if let Err(error) = conn.send_raw_line("this is not valid json {{{\n").await {
256        return CheckResult::fail(
257            CHECK_MALFORMED,
258            format!("provider closed its input on a malformed line: {error}"),
259        );
260    }
261    if let Err(error) = conn
262        .send(&ocp_host::Envelope::Query {
263            query: sample_query(),
264        })
265        .await
266    {
267        return CheckResult::fail(
268            CHECK_MALFORMED,
269            format!("provider died after a malformed line (before a valid query): {error}"),
270        );
271    }
272    match conn.recv().await {
273        Ok(ocp_host::Envelope::Frames { .. }) => CheckResult::pass(
274            CHECK_MALFORMED,
275            "provider ignored a malformed line and still answered a valid query",
276        ),
277        Ok(ocp_host::Envelope::Error { message }) => CheckResult::pass(
278            CHECK_MALFORMED,
279            format!("provider errored cleanly on malformed input and stayed alive: {message}"),
280        ),
281        Ok(other) => CheckResult::fail(
282            CHECK_MALFORMED,
283            format!(
284                "provider replied to a valid query with an unexpected `{}` envelope",
285                ocp_host::envelope_kind(&other)
286            ),
287        ),
288        Err(HostError::ProviderCrashed { .. }) => CheckResult::fail(
289            CHECK_MALFORMED,
290            "provider crashed on a malformed line — it must error-or-ignore, not die",
291        ),
292        Err(error) => CheckResult::fail(
293            CHECK_MALFORMED,
294            format!("provider mishandled malformed input: {error}"),
295        ),
296    }
297}
298
299/// The query the suite probes every provider with — no `kinds` filter, so any
300/// provider is asked for its best frames (§3.3).
301pub fn sample_query() -> ContextQuery {
302    ContextQuery {
303        goal: "conformance probe: return your most relevant frames".into(),
304        query_text: Some("conformance probe".into()),
305        embedding: None,
306        kinds: vec![],
307        anchors: vec![],
308        max_frames: 8,
309        max_tokens: 4096,
310        as_of: None,
311    }
312}
313
314/// Validate a query result's frames against the `ContextFrame` contract
315/// (§3.4). Returns `(passed, evidence)`. Zero frames is permitted — a
316/// provider may simply have nothing relevant.
317pub fn check_frames(result: &ContextQueryResult) -> (bool, String) {
318    if result.frames.is_empty() {
319        return (
320            true,
321            "provider returned 0 frames (permitted — nothing relevant to the probe)".into(),
322        );
323    }
324
325    let mut problems = Vec::new();
326    for (i, frame) in result.frames.iter().enumerate() {
327        if !frame.has_valid_score() {
328            problems.push(format!("frame[{i}] score {} is outside [0,1]", frame.score));
329        }
330        if frame.title.trim().is_empty() {
331            problems.push(format!("frame[{i}] has an empty title"));
332        }
333        match &frame.citation_label {
334            Some(label) if !label.trim().is_empty() => {}
335            _ => problems.push(format!(
336                "frame[{i}] is missing a citation_label (§3.4 — never a bare id)"
337            )),
338        }
339    }
340
341    if problems.is_empty() {
342        (
343            true,
344            format!(
345                "{} frame(s) — all scores in [0,1], titles + citation labels present",
346                result.frames.len()
347            ),
348        )
349    } else {
350        (false, problems.join("; "))
351    }
352}
353
354fn describe_handshake(info: &ProviderInfo, caps: &Capabilities) -> String {
355    format!(
356        "provider '{}' v{} — data-flow reads={} writes={} egress={}; query kinds={:?}, upsert={}, graph={}",
357        info.name,
358        info.version,
359        info.data_flow.reads,
360        info.data_flow.writes,
361        info.data_flow.egress,
362        caps.query.kinds,
363        caps.upsert,
364        caps.graph,
365    )
366}