chio_conformance/
runner.rs

1use std::env;
2use std::ffi::OsString;
3use std::fs;
4use std::net::{SocketAddr, TcpListener, TcpStream};
5use std::path::{Path, PathBuf};
6use std::process::{Child, Command, Stdio};
7use std::thread;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use crate::{
11    generate_markdown_report, load_results_from_dir, load_scenarios_from_dir, CompatibilityReport,
12};
13
14const SERVER_STARTUP_ATTEMPTS: usize = 900;
15const SERVER_STARTUP_POLL_INTERVAL: Duration = Duration::from_millis(100);
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum PeerTarget {
19    Js,
20    Python,
21    Go,
22    Cpp,
23}
24
25impl PeerTarget {
26    pub fn label(self) -> &'static str {
27        match self {
28            Self::Js => "js",
29            Self::Python => "python",
30            Self::Go => "go",
31            Self::Cpp => "cpp",
32        }
33    }
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub enum ConformanceAuthMode {
38    StaticBearer,
39    LocalOAuth,
40}
41
42#[derive(Debug, Clone)]
43pub struct ConformanceRunOptions {
44    pub repo_root: PathBuf,
45    pub scenarios_dir: PathBuf,
46    pub results_dir: PathBuf,
47    pub report_output: PathBuf,
48    pub policy_path: PathBuf,
49    pub upstream_server_script: PathBuf,
50    pub auth_mode: ConformanceAuthMode,
51    pub auth_token: String,
52    pub admin_token: String,
53    pub auth_scope: String,
54    pub listen: Option<SocketAddr>,
55    pub peers: Vec<PeerTarget>,
56    pub node_binary: OsString,
57    pub python_binary: OsString,
58    pub go_binary: OsString,
59    pub cargo_binary: OsString,
60    pub peer_binaries: Vec<(PeerTarget, PathBuf)>,
61}
62
63#[derive(Debug, Clone)]
64pub struct ConformanceRunSummary {
65    pub listen: SocketAddr,
66    pub results_dir: PathBuf,
67    pub report_output: PathBuf,
68    pub peer_result_files: Vec<PathBuf>,
69}
70
71struct ConformanceRuntimeState {
72    // Keep durable runtime state out of publishable artifacts and retain it
73    // until the child server has been stopped.
74    _directory: tempfile::TempDir,
75    auth_server_seed_path: PathBuf,
76    session_db_path: PathBuf,
77}
78
79impl ConformanceRuntimeState {
80    fn create() -> Result<Self, std::io::Error> {
81        let mut builder = tempfile::Builder::new();
82        builder.prefix("chio-conformance-runtime-");
83        #[cfg(unix)]
84        {
85            use std::os::unix::fs::PermissionsExt;
86            builder.permissions(fs::Permissions::from_mode(0o700));
87        }
88        let directory = builder.tempdir()?;
89        #[cfg(unix)]
90        {
91            use std::os::unix::fs::PermissionsExt;
92            fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700))?;
93        }
94        let auth_server_seed_path = directory.path().join("auth-server.seed");
95        let session_db_path = directory.path().join("mcp-session.sqlite3");
96        Ok(Self {
97            _directory: directory,
98            auth_server_seed_path,
99            session_db_path,
100        })
101    }
102}
103
104#[derive(Debug, thiserror::Error)]
105pub enum RunnerError {
106    #[error("i/o error: {0}")]
107    Io(#[from] std::io::Error),
108
109    #[error("failed to spawn process `{command}`: {source}")]
110    Spawn {
111        command: String,
112        #[source]
113        source: std::io::Error,
114    },
115
116    #[error("process `{command}` exited unsuccessfully with status {status}; see {log_path}")]
117    ProcessFailed {
118        command: String,
119        status: i32,
120        log_path: String,
121    },
122
123    #[error("timeout while waiting for MCP edge on {listen}")]
124    ServerStartupTimeout { listen: SocketAddr },
125
126    #[error("failed to load generated artifacts: {0}")]
127    Load(#[from] crate::load::LoadError),
128
129    #[error("peer result generation produced no JSON files in {path}")]
130    NoResults { path: String },
131}
132
133fn conformance_fixture_root_from_manifest_dir(manifest_dir: &Path) -> PathBuf {
134    let workspace_root = manifest_dir
135        .ancestors()
136        .nth(3)
137        .map(Path::to_path_buf)
138        .unwrap_or_else(|| manifest_dir.to_path_buf());
139    if workspace_root
140        .join("tests/conformance/scenarios/mcp_core")
141        .is_dir()
142        && workspace_root
143            .join("tests/conformance/fixtures/mcp_core/policy.yaml")
144            .is_file()
145    {
146        return workspace_root;
147    }
148
149    manifest_dir.to_path_buf()
150}
151
152pub fn default_repo_root() -> PathBuf {
153    conformance_fixture_root_from_manifest_dir(Path::new(env!("CARGO_MANIFEST_DIR")))
154}
155
156/// Read a conformance harness token from `var`, or return a local-only default.
157///
158/// When the env var is unset, the fallback uses the `dev-only-conformance-*`
159/// prefix plus a short random suffix. Set `CHIO_CONFORMANCE_AUTH_TOKEN` and
160/// `CHIO_CONFORMANCE_ADMIN_TOKEN` for CI or shared environments.
161fn conformance_token_from_env(var: &str, role: &str) -> String {
162    if let Ok(value) = env::var(var) {
163        let trimmed = value.trim();
164        if !trimmed.is_empty() {
165            return trimmed.to_string();
166        }
167    }
168    let nonce = SystemTime::now()
169        .duration_since(UNIX_EPOCH)
170        .map(|duration| duration.as_nanos())
171        .unwrap_or(0);
172    format!("dev-only-conformance-{role}-{nonce}")
173}
174
175/// Pass conformance tokens through child env vars so shared CI runners do not
176/// expose secrets via `/proc/<pid>/cmdline`.
177fn apply_conformance_auth_env(
178    command: &mut Command,
179    options: &ConformanceRunOptions,
180    auth_mode: ConformanceAuthMode,
181) {
182    command
183        .env("CHIO_CONFORMANCE_AUTH_TOKEN", &options.auth_token)
184        .env("CHIO_CONFORMANCE_ADMIN_TOKEN", &options.admin_token);
185    match auth_mode {
186        ConformanceAuthMode::StaticBearer => {
187            command.env("CHIO_AUTH_TOKEN", &options.auth_token);
188        }
189        ConformanceAuthMode::LocalOAuth => {
190            command
191                .env_remove("CHIO_AUTH_TOKEN")
192                .env_remove("CHIO_MCP_AUTH_TOKEN")
193                .env_remove("CHIO_MCP_ADMIN_TOKEN");
194            command.env("CHIO_ADMIN_TOKEN", &options.admin_token);
195        }
196    }
197}
198
199pub fn default_run_options() -> ConformanceRunOptions {
200    let repo_root = default_repo_root();
201    ConformanceRunOptions {
202        scenarios_dir: repo_root.join("tests/conformance/scenarios/mcp_core"),
203        results_dir: repo_root.join("tests/conformance/results/generated/mcp-core-live"),
204        report_output: repo_root.join("tests/conformance/reports/generated/mcp-core-live.md"),
205        policy_path: repo_root.join("tests/conformance/fixtures/mcp_core/policy.yaml"),
206        upstream_server_script: repo_root
207            .join("tests/conformance/fixtures/mcp_core/mock_mcp_server.py"),
208        auth_mode: ConformanceAuthMode::StaticBearer,
209        auth_token: conformance_token_from_env("CHIO_CONFORMANCE_AUTH_TOKEN", "auth"),
210        admin_token: conformance_token_from_env("CHIO_CONFORMANCE_ADMIN_TOKEN", "admin"),
211        auth_scope: "mcp:invoke".to_string(),
212        listen: None,
213        peers: vec![PeerTarget::Js, PeerTarget::Python],
214        node_binary: OsString::from("node"),
215        python_binary: OsString::from("python3"),
216        go_binary: OsString::from("go"),
217        cargo_binary: OsString::from("cargo"),
218        peer_binaries: Vec::new(),
219        repo_root,
220    }
221}
222
223pub fn run_conformance_harness(
224    options: &ConformanceRunOptions,
225) -> Result<ConformanceRunSummary, RunnerError> {
226    if options.results_dir.exists() {
227        fs::remove_dir_all(&options.results_dir)?;
228    }
229    fs::create_dir_all(&options.results_dir)?;
230    #[cfg(unix)]
231    {
232        use std::os::unix::fs::PermissionsExt;
233        fs::set_permissions(&options.results_dir, fs::Permissions::from_mode(0o700))?;
234    }
235    if let Some(parent) = options.report_output.parent() {
236        fs::create_dir_all(parent)?;
237    }
238
239    let artifacts_dir = options.results_dir.join("artifacts");
240    let logs_dir = artifacts_dir.join("logs");
241    fs::create_dir_all(&logs_dir)?;
242    #[cfg(unix)]
243    {
244        use std::os::unix::fs::PermissionsExt;
245        fs::set_permissions(&artifacts_dir, fs::Permissions::from_mode(0o700))?;
246    }
247
248    let listen = match options.listen {
249        Some(listen) => listen,
250        None => reserve_listen_addr()?,
251    };
252    let chio_executable = ensure_chio_executable(&options.repo_root, &options.cargo_binary)?;
253    let runtime_state = ConformanceRuntimeState::create()?;
254    let server_log_path = logs_dir.join("chio-mcp-serve-http.log");
255    let server = spawn_remote_edge(
256        &chio_executable,
257        options,
258        listen,
259        &runtime_state,
260        &server_log_path,
261    )?;
262    let mut server_guard = ChildGuard { child: server };
263    wait_for_server(listen, &mut server_guard.child, &server_log_path)?;
264
265    let mut peer_result_files = Vec::new();
266    for peer in &options.peers {
267        let peer_results_path = options
268            .results_dir
269            .join(format!("{}-remote-http.json", peer.label()));
270        let peer_artifacts_dir = artifacts_dir.join(peer.label());
271        let peer_log_path = logs_dir.join(format!("{}-peer.log", peer.label()));
272        fs::create_dir_all(&peer_artifacts_dir)?;
273        run_peer(
274            *peer,
275            options,
276            listen,
277            &peer_results_path,
278            &peer_artifacts_dir,
279            &peer_log_path,
280        )?;
281        peer_result_files.push(peer_results_path);
282    }
283
284    let results = load_results_from_dir(&options.results_dir)?;
285    if results.is_empty() {
286        return Err(RunnerError::NoResults {
287            path: options.results_dir.display().to_string(),
288        });
289    }
290    let report = CompatibilityReport {
291        scenarios: load_scenarios_from_dir(&options.scenarios_dir)?,
292        results,
293    };
294    fs::write(&options.report_output, generate_markdown_report(&report))?;
295
296    Ok(ConformanceRunSummary {
297        listen,
298        results_dir: options.results_dir.clone(),
299        report_output: options.report_output.clone(),
300        peer_result_files,
301    })
302}
303
304fn ensure_chio_executable(
305    repo_root: &Path,
306    cargo_binary: &OsString,
307) -> Result<PathBuf, RunnerError> {
308    let candidates = chio_executable_candidates(repo_root);
309    for candidate in &candidates {
310        if candidate.exists() {
311            return Ok(candidate.clone());
312        }
313    }
314    let status = Command::new(cargo_binary)
315        .current_dir(repo_root)
316        .arg("build")
317        .arg("-q")
318        .arg("-p")
319        .arg("chio-cli")
320        .status()
321        .map_err(|source| RunnerError::Spawn {
322            command: "cargo build -q -p chio-cli".to_string(),
323            source,
324        })?;
325    if !status.success() {
326        return Err(RunnerError::ProcessFailed {
327            command: "cargo build -q -p chio-cli".to_string(),
328            status: status.code().unwrap_or(1),
329            log_path: "<stderr>".to_string(),
330        });
331    }
332    for candidate in &candidates {
333        if candidate.exists() {
334            return Ok(candidate.clone());
335        }
336    }
337    let checked = candidates
338        .iter()
339        .map(|candidate| candidate.display().to_string())
340        .collect::<Vec<_>>()
341        .join(", ");
342    Err(RunnerError::ProcessFailed {
343        command: "cargo build -q -p chio-cli".to_string(),
344        status: 1,
345        log_path: format!("<stderr>; checked {checked}"),
346    })
347}
348
349fn chio_binary_name() -> &'static str {
350    if cfg!(windows) {
351        "chio.exe"
352    } else {
353        "chio"
354    }
355}
356
357fn chio_executable_candidates(repo_root: &Path) -> Vec<PathBuf> {
358    let mut candidates = Vec::new();
359    if let Ok(current_exe) = env::current_exe() {
360        if current_exe.file_name().and_then(|name| name.to_str()) == Some(chio_binary_name()) {
361            push_chio_candidate(&mut candidates, current_exe);
362        }
363    }
364    if let Some(target_dir) = env::var_os("CARGO_TARGET_DIR") {
365        push_chio_candidate(
366            &mut candidates,
367            PathBuf::from(target_dir)
368                .join("debug")
369                .join(chio_binary_name()),
370        );
371    }
372    push_chio_candidate(
373        &mut candidates,
374        repo_root
375            .join("target")
376            .join("debug")
377            .join(chio_binary_name()),
378    );
379    candidates
380}
381
382fn push_chio_candidate(candidates: &mut Vec<PathBuf>, path: PathBuf) {
383    if !candidates.iter().any(|candidate| candidate == &path) {
384        candidates.push(path);
385    }
386}
387
388fn spawn_remote_edge(
389    chio_executable: &Path,
390    options: &ConformanceRunOptions,
391    listen: SocketAddr,
392    runtime_state: &ConformanceRuntimeState,
393    log_path: &Path,
394) -> Result<Child, RunnerError> {
395    let log = fs::File::create(log_path)?;
396    let log_clone = log.try_clone()?;
397    let mut command = Command::new(chio_executable);
398    command
399        .current_dir(&options.repo_root)
400        .arg("mcp")
401        .arg("serve-http")
402        .arg("--policy")
403        .arg(&options.policy_path)
404        .arg("--server-id")
405        .arg("conformance-mcp-core")
406        .arg("--server-name")
407        .arg("Conformance Fixture")
408        .arg("--server-version")
409        .arg("0.1.0")
410        .arg("--listen")
411        .arg(listen.to_string());
412
413    let public_base_url = format!("http://{listen}");
414    command
415        .arg("--session-db")
416        .arg(&runtime_state.session_db_path);
417    let mut command_description = format!(
418        "{} mcp serve-http --policy {} --server-id conformance-mcp-core --listen {} --session-db {}",
419        chio_executable.display(),
420        options.policy_path.display(),
421        listen,
422        runtime_state.session_db_path.display()
423    );
424
425    apply_conformance_auth_env(&mut command, options, options.auth_mode);
426
427    match options.auth_mode {
428        ConformanceAuthMode::StaticBearer => {
429            command_description.push_str(" (auth via CHIO_AUTH_TOKEN env)");
430        }
431        ConformanceAuthMode::LocalOAuth => {
432            command
433                .arg("--public-base-url")
434                .arg(&public_base_url)
435                .arg("--auth-server-seed-file")
436                .arg(&runtime_state.auth_server_seed_path)
437                .arg("--auth-jwt-audience")
438                .arg(format!("{public_base_url}/mcp"))
439                .arg("--auth-scope")
440                .arg(&options.auth_scope);
441            command_description.push_str(&format!(
442                " --public-base-url {} --auth-server-seed-file {} --auth-jwt-audience {}/mcp --auth-scope {} (admin via CHIO_ADMIN_TOKEN env)",
443                public_base_url,
444                runtime_state.auth_server_seed_path.display(),
445                public_base_url,
446                options.auth_scope
447            ));
448        }
449    }
450
451    command
452        .arg("--")
453        .arg(&options.python_binary)
454        .arg(&options.upstream_server_script)
455        .stdout(Stdio::from(log))
456        .stderr(Stdio::from(log_clone))
457        .spawn()
458        .map_err(|source| RunnerError::Spawn {
459            command: format!(
460                "{} -- {} {}",
461                command_description,
462                PathBuf::from(&options.python_binary).display(),
463                options.upstream_server_script.display()
464            ),
465            source,
466        })
467}
468
469fn run_peer(
470    peer: PeerTarget,
471    options: &ConformanceRunOptions,
472    listen: SocketAddr,
473    results_output: &Path,
474    artifacts_dir: &Path,
475    log_path: &Path,
476) -> Result<(), RunnerError> {
477    let log = fs::File::create(log_path)?;
478    let log_clone = log.try_clone()?;
479    let base_url = format!("http://{listen}");
480    let command_description;
481    let mut command = if let Some(binary) = peer_binary_override(peer, options) {
482        command_description = binary.display().to_string();
483        let mut command = Command::new(binary);
484        command.current_dir(&options.repo_root);
485        command
486    } else {
487        match peer {
488            PeerTarget::Js => {
489                let script = options
490                    .repo_root
491                    .join("tests/conformance/peers/js/client.mjs");
492                command_description = format!(
493                    "{} {}",
494                    PathBuf::from(&options.node_binary).display(),
495                    script.display()
496                );
497                let mut command = Command::new(&options.node_binary);
498                command.current_dir(&options.repo_root).arg(script);
499                command
500            }
501            PeerTarget::Python => {
502                let script = options
503                    .repo_root
504                    .join("tests/conformance/peers/python/client.py");
505                command_description = format!(
506                    "{} {}",
507                    PathBuf::from(&options.python_binary).display(),
508                    script.display()
509                );
510                let mut command = Command::new(&options.python_binary);
511                command.current_dir(&options.repo_root).arg(script);
512                command
513            }
514            PeerTarget::Go => {
515                command_description = format!(
516                    "{} run ./cmd/conformance-peer",
517                    PathBuf::from(&options.go_binary).display()
518                );
519                let mut command = Command::new(&options.go_binary);
520                command
521                    .current_dir(options.repo_root.join("sdks/go/chio-go"))
522                    .arg("run")
523                    .arg("./cmd/conformance-peer");
524                command
525            }
526            PeerTarget::Cpp => {
527                let executable = ensure_cpp_peer_executable(&options.repo_root)?;
528                command_description = executable.display().to_string();
529                let mut command = Command::new(executable);
530                command.current_dir(&options.repo_root);
531                command
532            }
533        }
534    };
535
536    apply_conformance_auth_env(&mut command, options, options.auth_mode);
537
538    let status = command
539        .arg("--base-url")
540        .arg(base_url)
541        .arg("--auth-mode")
542        .arg(match options.auth_mode {
543            ConformanceAuthMode::StaticBearer => "static-bearer",
544            ConformanceAuthMode::LocalOAuth => "oauth-local",
545        })
546        .arg("--auth-scope")
547        .arg(&options.auth_scope)
548        .arg("--scenarios-dir")
549        .arg(&options.scenarios_dir)
550        .arg("--results-output")
551        .arg(results_output)
552        .arg("--artifacts-dir")
553        .arg(artifacts_dir)
554        .stdout(Stdio::from(log))
555        .stderr(Stdio::from(log_clone))
556        .status()
557        .map_err(|source| RunnerError::Spawn {
558            command: format!(
559                "{} --base-url http://{} --scenarios-dir {} --results-output {}",
560                command_description,
561                listen,
562                options.scenarios_dir.display(),
563                results_output.display()
564            ),
565            source,
566        })?;
567
568    if !status.success() {
569        return Err(RunnerError::ProcessFailed {
570            command: command_description,
571            status: status.code().unwrap_or(1),
572            log_path: log_path.display().to_string(),
573        });
574    }
575    Ok(())
576}
577
578fn peer_binary_override(peer: PeerTarget, options: &ConformanceRunOptions) -> Option<&Path> {
579    options
580        .peer_binaries
581        .iter()
582        .find_map(|(target, path)| (*target == peer).then_some(path.as_path()))
583}
584
585fn ensure_cpp_peer_executable(repo_root: &Path) -> Result<PathBuf, RunnerError> {
586    let build_dir = repo_root.join("target/chio-cpp-conformance");
587    let build_config = "Debug";
588    let executable_name = format!("chio_cpp_conformance_peer{}", std::env::consts::EXE_SUFFIX);
589    let executable_candidates = [
590        build_dir.join(&executable_name),
591        build_dir.join(build_config).join(&executable_name),
592    ];
593    let source_dir = repo_root.join("sdks/cpp/chio-cpp");
594    let configure_status = Command::new("cmake")
595        .current_dir(repo_root)
596        .arg("-S")
597        .arg(&source_dir)
598        .arg("-B")
599        .arg(&build_dir)
600        .arg("-DCHIO_CPP_BUILD_TESTS=OFF")
601        .arg("-DCHIO_CPP_BUILD_EXAMPLES=OFF")
602        .arg("-DCHIO_CPP_ENABLE_CURL=ON")
603        .arg("-DCHIO_CPP_BUILD_CONFORMANCE_PEER=ON")
604        .arg("-DCMAKE_BUILD_TYPE=Debug")
605        .status()
606        .map_err(|source| RunnerError::Spawn {
607            command: "cmake configure chio_cpp_conformance_peer".to_string(),
608            source,
609        })?;
610    if !configure_status.success() {
611        return Err(RunnerError::ProcessFailed {
612            command: "cmake configure chio_cpp_conformance_peer".to_string(),
613            status: configure_status.code().unwrap_or(1),
614            log_path: "<stderr>".to_string(),
615        });
616    }
617
618    let build_status = Command::new("cmake")
619        .current_dir(repo_root)
620        .arg("--build")
621        .arg(&build_dir)
622        .arg("--target")
623        .arg("chio_cpp_conformance_peer")
624        .arg("--config")
625        .arg(build_config)
626        .status()
627        .map_err(|source| RunnerError::Spawn {
628            command: "cmake --build chio_cpp_conformance_peer".to_string(),
629            source,
630        })?;
631    if !build_status.success() {
632        return Err(RunnerError::ProcessFailed {
633            command: "cmake --build chio_cpp_conformance_peer".to_string(),
634            status: build_status.code().unwrap_or(1),
635            log_path: "<stderr>".to_string(),
636        });
637    }
638
639    for executable in executable_candidates {
640        if executable.exists() {
641            return Ok(executable);
642        }
643    }
644    Err(RunnerError::ProcessFailed {
645        command: "cmake --build chio_cpp_conformance_peer".to_string(),
646        status: 1,
647        log_path: "<stderr>".to_string(),
648    })
649}
650
651fn reserve_listen_addr() -> Result<SocketAddr, RunnerError> {
652    let listener = TcpListener::bind("127.0.0.1:0")?;
653    let addr = listener.local_addr()?;
654    drop(listener);
655    Ok(addr)
656}
657
658fn wait_for_server(
659    listen: SocketAddr,
660    server: &mut Child,
661    log_path: &Path,
662) -> Result<(), RunnerError> {
663    for _ in 0..SERVER_STARTUP_ATTEMPTS {
664        if TcpStream::connect(listen).is_ok() {
665            thread::sleep(SERVER_STARTUP_POLL_INTERVAL);
666            return Ok(());
667        }
668        if let Some(status) = server.try_wait()? {
669            return Err(RunnerError::ProcessFailed {
670                command: "chio mcp serve-http".to_string(),
671                status: status.code().unwrap_or(1),
672                log_path: log_path.display().to_string(),
673            });
674        }
675        thread::sleep(SERVER_STARTUP_POLL_INTERVAL);
676    }
677    Err(RunnerError::ServerStartupTimeout { listen })
678}
679
680struct ChildGuard {
681    child: Child,
682}
683
684impl Drop for ChildGuard {
685    fn drop(&mut self) {
686        let _ = self.child.kill();
687        let _ = self.child.wait();
688    }
689}
690
691pub fn unique_run_dir(prefix: &str) -> PathBuf {
692    let nonce = SystemTime::now()
693        .duration_since(UNIX_EPOCH)
694        .map(|duration| duration.as_nanos())
695        .unwrap_or(0);
696    std::env::temp_dir().join(format!("{prefix}-{nonce}"))
697}
698
699#[cfg(test)]
700mod tests {
701    use super::{
702        apply_conformance_auth_env, conformance_fixture_root_from_manifest_dir,
703        default_run_options, ConformanceAuthMode, ConformanceRuntimeState,
704    };
705    use std::ffi::OsStr;
706    use std::fs;
707    use std::path::Path;
708    use std::process::Command;
709    use std::time::{SystemTime, UNIX_EPOCH};
710
711    fn command_env(command: &Command, key: &str) -> Option<Option<String>> {
712        command.get_envs().find_map(|(name, value)| {
713            if name == OsStr::new(key) {
714                return Some(value.map(|raw| raw.to_string_lossy().into_owned()));
715            }
716            None
717        })
718    }
719
720    fn unique_test_dir(label: &str) -> std::path::PathBuf {
721        let nanos = SystemTime::now()
722            .duration_since(UNIX_EPOCH)
723            .map(|duration| duration.as_nanos())
724            .unwrap_or(0);
725        std::env::temp_dir().join(format!(
726            "chio-conformance-{label}-{}-{nanos}",
727            std::process::id()
728        ))
729    }
730
731    fn create_default_fixture_tree(root: &Path) {
732        let scenarios = root.join("tests/conformance/scenarios/mcp_core");
733        let fixtures = root.join("tests/conformance/fixtures/mcp_core");
734        if let Err(error) = fs::create_dir_all(&scenarios) {
735            panic!("failed to create {}: {error}", scenarios.display());
736        }
737        if let Err(error) = fs::create_dir_all(&fixtures) {
738            panic!("failed to create {}: {error}", fixtures.display());
739        }
740        if let Err(error) = fs::write(scenarios.join("initialize.json"), "{}\n") {
741            panic!("failed to write scenario fixture: {error}");
742        }
743        if let Err(error) = fs::write(fixtures.join("policy.yaml"), "version: 1\n") {
744            panic!("failed to write policy fixture: {error}");
745        }
746    }
747
748    #[test]
749    fn runtime_state_is_private_and_removed_after_the_run() {
750        let runtime_state = ConformanceRuntimeState::create()
751            .unwrap_or_else(|error| panic!("create conformance runtime state: {error}"));
752        let runtime_root = runtime_state._directory.path().to_path_buf();
753        assert_eq!(
754            runtime_state.session_db_path.parent(),
755            Some(runtime_root.as_path())
756        );
757        assert_eq!(
758            runtime_state.auth_server_seed_path.parent(),
759            Some(runtime_root.as_path())
760        );
761        #[cfg(unix)]
762        {
763            use std::os::unix::fs::PermissionsExt;
764            let metadata = fs::metadata(&runtime_root)
765                .unwrap_or_else(|error| panic!("read runtime directory metadata: {error}"));
766            assert_eq!(metadata.permissions().mode() & 0o777, 0o700);
767        }
768
769        drop(runtime_state);
770        assert!(!runtime_root.exists());
771    }
772
773    #[test]
774    fn store_less_fixture_opts_into_ephemeral_revocation() {
775        // The harness launches `chio mcp serve-http` with no `--revocation-db`,
776        // so the kernel's revocation store is the in-memory default. Durable
777        // persistence is the default posture, so a mediated tool call is denied
778        // by the revocation durability gate unless the policy opts into an
779        // ephemeral revocation store. A fixture that opts into an ephemeral
780        // receipt log (the store-less scaffold posture) must therefore also opt
781        // into an ephemeral revocation store, or the whole suite denies before
782        // it can run.
783        #[derive(serde::Deserialize)]
784        struct KernelSection {
785            #[serde(default)]
786            allow_ephemeral_receipt_log: bool,
787            #[serde(default)]
788            allow_ephemeral_revocation_store: bool,
789        }
790        #[derive(serde::Deserialize)]
791        struct FixturePolicy {
792            kernel: KernelSection,
793        }
794
795        let fixture_root =
796            conformance_fixture_root_from_manifest_dir(Path::new(env!("CARGO_MANIFEST_DIR")));
797        let policy_path = fixture_root.join("tests/conformance/fixtures/mcp_core/policy.yaml");
798        let raw = fs::read_to_string(&policy_path)
799            .unwrap_or_else(|error| panic!("read {}: {error}", policy_path.display()));
800        let policy: FixturePolicy = serde_yaml::from_str(&raw)
801            .unwrap_or_else(|error| panic!("parse {}: {error}", policy_path.display()));
802
803        if policy.kernel.allow_ephemeral_receipt_log {
804            assert!(
805                policy.kernel.allow_ephemeral_revocation_store,
806                "a store-less conformance fixture that allows an ephemeral receipt log must also \
807                 allow an ephemeral revocation store, or mediated tool calls are denied by the \
808                 revocation durability gate"
809            );
810        }
811    }
812
813    #[test]
814    fn fixture_root_prefers_workspace_tree_when_present() {
815        let root = unique_test_dir("workspace");
816        let manifest_dir = root.join("crates/tooling/chio-conformance");
817        create_default_fixture_tree(&root);
818        if let Err(error) = fs::create_dir_all(&manifest_dir) {
819            panic!("failed to create {}: {error}", manifest_dir.display());
820        }
821
822        assert_eq!(
823            conformance_fixture_root_from_manifest_dir(&manifest_dir),
824            root
825        );
826
827        let _ = fs::remove_dir_all(root);
828    }
829
830    #[test]
831    fn fixture_root_falls_back_to_packaged_crate_tree() {
832        let package_root = unique_test_dir("package");
833        create_default_fixture_tree(&package_root);
834
835        assert_eq!(
836            conformance_fixture_root_from_manifest_dir(&package_root),
837            package_root
838        );
839
840        let _ = fs::remove_dir_all(package_root);
841    }
842
843    #[test]
844    fn local_oauth_child_env_removes_inherited_static_bearer_token() {
845        let mut options = default_run_options();
846        options.auth_mode = ConformanceAuthMode::LocalOAuth;
847        options.auth_token = "local-auth-token".to_string();
848        options.admin_token = "local-admin-token".to_string();
849        let mut command = Command::new("chio");
850        command.env("CHIO_AUTH_TOKEN", "parent-static-token");
851        command.env("CHIO_MCP_AUTH_TOKEN", "parent-mcp-static-token");
852        command.env("CHIO_MCP_ADMIN_TOKEN", "parent-mcp-admin-token");
853
854        apply_conformance_auth_env(&mut command, &options, options.auth_mode);
855
856        assert_eq!(command_env(&command, "CHIO_AUTH_TOKEN"), Some(None));
857        assert_eq!(command_env(&command, "CHIO_MCP_AUTH_TOKEN"), Some(None));
858        assert_eq!(command_env(&command, "CHIO_MCP_ADMIN_TOKEN"), Some(None));
859        assert_eq!(
860            command_env(&command, "CHIO_ADMIN_TOKEN"),
861            Some(Some("local-admin-token".to_string()))
862        );
863        assert_eq!(
864            command_env(&command, "CHIO_CONFORMANCE_AUTH_TOKEN"),
865            Some(Some("local-auth-token".to_string()))
866        );
867        assert_eq!(
868            command_env(&command, "CHIO_CONFORMANCE_ADMIN_TOKEN"),
869            Some(Some("local-admin-token".to_string()))
870        );
871    }
872}
chio_conformance/runner.rs

chio_conformance/
runner.rs
