1use ocp_host::{ConsentRecord, ContextProvider, Host, HostError, RawStdioConnection};
31use ocp_types::{Capabilities, ContextQuery, ContextQueryResult, ProviderInfo};
32
33mod report;
34
35pub use report::{CheckResult, CheckStatus, ConformanceReport};
36
37pub const CHECK_HANDSHAKE: &str = "handshake";
39pub const CHECK_FRAME_VALIDITY: &str = "frame-validity";
40pub const CHECK_BUDGET_HONESTY: &str = "budget-honesty";
41pub const CHECK_SHUTDOWN: &str = "shutdown-clean";
42pub const CHECK_MALFORMED: &str = "malformed-input-tolerance";
43
44pub enum ProviderTarget {
47 Stdio { program: String, args: Vec<String> },
49 Http { url: String },
51 InProcess(Box<dyn ContextProvider>),
53}
54
55impl ProviderTarget {
56 pub fn describe(&self) -> String {
58 match self {
59 ProviderTarget::Stdio { program, args } => {
60 if args.is_empty() {
61 format!("stdio: {program}")
62 } else {
63 format!("stdio: {program} {}", args.join(" "))
64 }
65 }
66 ProviderTarget::Http { url } => format!("http: {url}"),
67 ProviderTarget::InProcess(provider) => format!("in-process: {}", provider.id()),
68 }
69 }
70}
71
72pub async fn run_conformance(target: ProviderTarget) -> ConformanceReport {
76 let description = target.describe();
77
78 let stdio_probe = match &target {
81 ProviderTarget::Stdio { program, args } => Some((program.clone(), args.clone())),
82 _ => None,
83 };
84
85 let mut checks = Vec::new();
86
87 match build_host(target).await {
88 Ok((host, id, info, caps)) => {
89 if info.name.trim().is_empty() || info.version.trim().is_empty() {
90 checks.push(CheckResult::fail(
91 CHECK_HANDSHAKE,
92 format!(
93 "provider identity incomplete: name='{}' version='{}'",
94 info.name, info.version
95 ),
96 ));
97 } else {
98 checks.push(CheckResult::pass(
99 CHECK_HANDSHAKE,
100 describe_handshake(&info, &caps),
101 ));
102 }
103 run_query_and_shutdown_checks(host, &id, &mut checks).await;
104 }
105 Err(error) => {
106 checks.push(CheckResult::fail(
107 CHECK_HANDSHAKE,
108 format!("could not establish provider: {error}"),
109 ));
110 for name in [CHECK_FRAME_VALIDITY, CHECK_BUDGET_HONESTY, CHECK_SHUTDOWN] {
111 checks.push(CheckResult::skip(name, "handshake failed"));
112 }
113 }
114 }
115
116 match stdio_probe {
117 Some((program, args)) => checks.push(malformed_stdio_probe(&program, &args).await),
118 None => checks.push(CheckResult::skip(
119 CHECK_MALFORMED,
120 "wire-level malformed-input probe applies to stdio providers only",
121 )),
122 }
123
124 ConformanceReport {
125 target: description,
126 checks,
127 }
128}
129
130async fn build_host(
135 target: ProviderTarget,
136) -> Result<(Host, String, ProviderInfo, Capabilities), HostError> {
137 let mut host = Host::new();
138 let (id, info, caps) = match target {
139 ProviderTarget::Stdio { program, args } => {
140 let id = "provider-under-test".to_string();
141 host.add_stdio(id.clone(), &program, &args).await?;
142 capture_identity(&host, &id)?
143 }
144 ProviderTarget::Http { url } => {
145 let id = "provider-under-test".to_string();
146 host.add_http(id.clone(), url).await?;
147 capture_identity(&host, &id)?
148 }
149 ProviderTarget::InProcess(provider) => {
150 let id = provider.id().to_string();
151 let info = provider.info().clone();
152 let caps = provider.capabilities().clone();
153 host.register(provider);
154 (id, info, caps)
155 }
156 };
157
158 if info.data_flow.egress {
159 host.record_consent(ConsentRecord::new(
160 id.clone(),
161 info.data_flow,
162 "conformance run under test",
163 ));
164 }
165
166 Ok((host, id, info, caps))
167}
168
169fn capture_identity(
170 host: &Host,
171 id: &str,
172) -> Result<(String, ProviderInfo, Capabilities), HostError> {
173 let provider = host
174 .provider(id)
175 .ok_or_else(|| HostError::UnknownProvider(id.to_string()))?;
176 Ok((
177 id.to_string(),
178 provider.info().clone(),
179 provider.capabilities().clone(),
180 ))
181}
182
183async fn run_query_and_shutdown_checks(host: Host, id: &str, checks: &mut Vec<CheckResult>) {
184 let query = sample_query();
185 match host.query_provider(id, &query).await {
186 Ok(result) => {
187 let (ok, evidence) = check_frames(&result);
188 checks.push(CheckResult::from_bool(CHECK_FRAME_VALIDITY, ok, evidence));
189
190 if result.respects_budget(query.max_tokens) {
191 checks.push(CheckResult::pass(
192 CHECK_BUDGET_HONESTY,
193 format!(
194 "{} frame(s) sum to {} token(s), within the {}-token budget",
195 result.frames.len(),
196 result.total_token_cost(),
197 query.max_tokens
198 ),
199 ));
200 } else {
201 checks.push(CheckResult::fail(
202 CHECK_BUDGET_HONESTY,
203 format!(
204 "frames claim {} token(s), over the {}-token budget — token_cost is dishonest",
205 result.total_token_cost(),
206 query.max_tokens
207 ),
208 ));
209 }
210 }
211 Err(error) => {
212 let evidence = format!("query failed: {error}");
213 checks.push(CheckResult::fail(CHECK_FRAME_VALIDITY, evidence.clone()));
214 checks.push(CheckResult::fail(CHECK_BUDGET_HONESTY, evidence));
215 }
216 }
217
218 let results = host.shutdown().await;
219 match results.iter().find(|(pid, _)| pid == id) {
220 Some((_, Ok(()))) => checks.push(CheckResult::pass(
221 CHECK_SHUTDOWN,
222 "provider acknowledged shutdown and tore down cleanly",
223 )),
224 Some((_, Err(error))) => checks.push(CheckResult::fail(
225 CHECK_SHUTDOWN,
226 format!("shutdown error: {error}"),
227 )),
228 None => checks.push(CheckResult::fail(
229 CHECK_SHUTDOWN,
230 "provider vanished before shutdown could be attempted",
231 )),
232 }
233}
234
235async fn malformed_stdio_probe(program: &str, args: &[String]) -> CheckResult {
240 let mut conn = match RawStdioConnection::spawn(program, args).await {
241 Ok(conn) => conn,
242 Err(error) => {
243 return CheckResult::fail(
244 CHECK_MALFORMED,
245 format!("could not spawn provider: {error}"),
246 );
247 }
248 };
249 if let Err(error) = conn.handshake().await {
250 return CheckResult::fail(
251 CHECK_MALFORMED,
252 format!("handshake failed before the probe could run: {error}"),
253 );
254 }
255 if let Err(error) = conn.send_raw_line("this is not valid json {{{\n").await {
256 return CheckResult::fail(
257 CHECK_MALFORMED,
258 format!("provider closed its input on a malformed line: {error}"),
259 );
260 }
261 if let Err(error) = conn
262 .send(&ocp_host::Envelope::Query {
263 query: sample_query(),
264 })
265 .await
266 {
267 return CheckResult::fail(
268 CHECK_MALFORMED,
269 format!("provider died after a malformed line (before a valid query): {error}"),
270 );
271 }
272 match conn.recv().await {
273 Ok(ocp_host::Envelope::Frames { .. }) => CheckResult::pass(
274 CHECK_MALFORMED,
275 "provider ignored a malformed line and still answered a valid query",
276 ),
277 Ok(ocp_host::Envelope::Error { message }) => CheckResult::pass(
278 CHECK_MALFORMED,
279 format!("provider errored cleanly on malformed input and stayed alive: {message}"),
280 ),
281 Ok(other) => CheckResult::fail(
282 CHECK_MALFORMED,
283 format!(
284 "provider replied to a valid query with an unexpected `{}` envelope",
285 ocp_host::envelope_kind(&other)
286 ),
287 ),
288 Err(HostError::ProviderCrashed { .. }) => CheckResult::fail(
289 CHECK_MALFORMED,
290 "provider crashed on a malformed line — it must error-or-ignore, not die",
291 ),
292 Err(error) => CheckResult::fail(
293 CHECK_MALFORMED,
294 format!("provider mishandled malformed input: {error}"),
295 ),
296 }
297}
298
299pub fn sample_query() -> ContextQuery {
302 ContextQuery {
303 goal: "conformance probe: return your most relevant frames".into(),
304 query_text: Some("conformance probe".into()),
305 embedding: None,
306 kinds: vec![],
307 anchors: vec![],
308 max_frames: 8,
309 max_tokens: 4096,
310 as_of: None,
311 }
312}
313
314pub fn check_frames(result: &ContextQueryResult) -> (bool, String) {
318 if result.frames.is_empty() {
319 return (
320 true,
321 "provider returned 0 frames (permitted — nothing relevant to the probe)".into(),
322 );
323 }
324
325 let mut problems = Vec::new();
326 for (i, frame) in result.frames.iter().enumerate() {
327 if !frame.has_valid_score() {
328 problems.push(format!("frame[{i}] score {} is outside [0,1]", frame.score));
329 }
330 if frame.title.trim().is_empty() {
331 problems.push(format!("frame[{i}] has an empty title"));
332 }
333 match &frame.citation_label {
334 Some(label) if !label.trim().is_empty() => {}
335 _ => problems.push(format!(
336 "frame[{i}] is missing a citation_label (§3.4 — never a bare id)"
337 )),
338 }
339 }
340
341 if problems.is_empty() {
342 (
343 true,
344 format!(
345 "{} frame(s) — all scores in [0,1], titles + citation labels present",
346 result.frames.len()
347 ),
348 )
349 } else {
350 (false, problems.join("; "))
351 }
352}
353
354fn describe_handshake(info: &ProviderInfo, caps: &Capabilities) -> String {
355 format!(
356 "provider '{}' v{} — data-flow reads={} writes={} egress={}; query kinds={:?}, upsert={}, graph={}",
357 info.name,
358 info.version,
359 info.data_flow.reads,
360 info.data_flow.writes,
361 info.data_flow.egress,
362 caps.query.kinds,
363 caps.upsert,
364 caps.graph,
365 )
366}