1use std::fs;
2use std::path::{Path, PathBuf};
3
4use serde::de::DeserializeOwned;
5
6use crate::{ScenarioDescriptor, ScenarioResult};
7
8#[derive(Debug, thiserror::Error)]
9pub enum LoadError {
10 #[error("i/o error: {0}")]
11 Io(#[from] std::io::Error),
12
13 #[error("json error in {path}: {source}")]
14 Json {
15 path: String,
16 #[source]
17 source: serde_json::Error,
18 },
19}
20
21pub fn load_scenarios_from_dir(
22 path: impl AsRef<Path>,
23) -> Result<Vec<ScenarioDescriptor>, LoadError> {
24 let path = path.as_ref();
25 let mut scenarios = read_json_files::<ScenarioDescriptor>(path)?;
26 if scenarios.is_empty() {
27 return Err(empty_scenario_directory_error(path));
28 }
29 scenarios.sort_by(|left, right| left.id.cmp(&right.id));
30 Ok(scenarios)
31}
32
33pub fn load_results_from_dir(path: impl AsRef<Path>) -> Result<Vec<ScenarioResult>, LoadError> {
34 let mut results = Vec::new();
35 for file_path in collect_json_files(path.as_ref())? {
36 let content = fs::read_to_string(&file_path)?;
37 match serde_json::from_str::<Vec<ScenarioResult>>(&content) {
38 Ok(mut batch) => results.append(&mut batch),
39 Err(_) => {
40 let record = deserialize::<ScenarioResult>(&file_path, &content)?;
41 results.push(record);
42 }
43 }
44 }
45 results.sort_by(|left, right| {
46 left.scenario_id
47 .cmp(&right.scenario_id)
48 .then_with(|| left.peer.cmp(&right.peer))
49 .then_with(|| left.deployment_mode.cmp(&right.deployment_mode))
50 .then_with(|| left.transport.cmp(&right.transport))
51 });
52 Ok(results)
53}
54
55fn read_json_files<T: DeserializeOwned>(path: &Path) -> Result<Vec<T>, LoadError> {
56 let mut items = Vec::new();
57 for file_path in collect_json_files(path)? {
58 let content = fs::read_to_string(&file_path)?;
59 items.push(deserialize::<T>(&file_path, &content)?);
60 }
61 Ok(items)
62}
63
64fn deserialize<T: DeserializeOwned>(path: &Path, content: &str) -> Result<T, LoadError> {
65 serde_json::from_str(content).map_err(|source| LoadError::Json {
66 path: path.display().to_string(),
67 source,
68 })
69}
70
71fn collect_json_files(path: &Path) -> Result<Vec<PathBuf>, LoadError> {
72 require_json_directory(path)?;
73 let mut files = Vec::new();
74 walk_json_files(path, &mut files)?;
75 files.sort();
76 Ok(files)
77}
78
79fn empty_scenario_directory_error(path: &Path) -> LoadError {
80 LoadError::Io(std::io::Error::new(
81 std::io::ErrorKind::InvalidData,
82 format!(
83 "conformance scenario directory {} is empty: expected at least one JSON scenario",
84 path.display()
85 ),
86 ))
87}
88
89fn require_json_directory(path: &Path) -> Result<(), LoadError> {
90 let metadata = fs::symlink_metadata(path).map_err(|source| {
91 LoadError::Io(std::io::Error::new(
92 source.kind(),
93 format!(
94 "conformance JSON directory {} is not readable: {source}",
95 path.display()
96 ),
97 ))
98 })?;
99 let file_type = metadata.file_type();
100 if file_type.is_symlink() {
101 return Err(LoadError::Io(std::io::Error::other(format!(
102 "refusing symlinked conformance fixture directory: {}",
103 path.display()
104 ))));
105 }
106 if !file_type.is_dir() {
107 return Err(LoadError::Io(std::io::Error::new(
108 std::io::ErrorKind::InvalidInput,
109 format!(
110 "conformance JSON directory {} is not a directory",
111 path.display()
112 ),
113 )));
114 }
115 Ok(())
116}
117
118fn walk_json_files(path: &Path, files: &mut Vec<PathBuf>) -> Result<(), LoadError> {
119 for entry in fs::read_dir(path)? {
120 let entry = entry?;
121 let entry_path = entry.path();
122 let file_type = entry.file_type()?;
123 if file_type.is_symlink() {
124 return Err(LoadError::Io(std::io::Error::other(format!(
125 "refusing symlink in conformance fixture tree: {}",
126 entry_path.display()
127 ))));
128 }
129 if file_type.is_dir() {
130 walk_json_files(&entry_path, files)?;
131 } else if file_type.is_file()
132 && entry_path.extension().and_then(|value| value.to_str()) == Some("json")
133 {
134 files.push(entry_path);
135 }
136 }
137 Ok(())
138}
139
140#[cfg(test)]
141mod tests {
142 use std::time::{SystemTime, UNIX_EPOCH};
143
144 use super::*;
145
146 fn unique_dir(prefix: &str) -> Result<PathBuf, LoadError> {
147 let nonce = SystemTime::now()
148 .duration_since(UNIX_EPOCH)
149 .map_err(std::io::Error::other)?
150 .as_nanos();
151 Ok(std::env::temp_dir().join(format!("{prefix}-{nonce}")))
152 }
153
154 #[test]
155 fn loads_scenarios_and_results_from_json_directories() -> Result<(), LoadError> {
156 let dir = unique_dir("chio-conformance-load")?;
157 fs::create_dir_all(dir.join("scenarios"))?;
158 fs::create_dir_all(dir.join("results"))?;
159
160 fs::write(
161 dir.join("scenarios/initialize.json"),
162 r#"{
163 "id": "initialize",
164 "title": "Initialize",
165 "area": "lifecycle",
166 "category": "mcp-core",
167 "specVersions": ["2025-11-25"],
168 "transport": ["stdio"],
169 "peerRoles": ["client_to_chio_server"],
170 "deploymentModes": ["wrapped_stdio"],
171 "requiredCapabilities": {"server": [], "client": []},
172 "tags": ["mcp-core"],
173 "expected": "pass"
174 }"#,
175 )?;
176 fs::write(
177 dir.join("results/results.json"),
178 r#"[{
179 "scenarioId": "initialize",
180 "peer": "js",
181 "peerRole": "client_to_chio_server",
182 "deploymentMode": "wrapped_stdio",
183 "transport": "stdio",
184 "specVersion": "2025-11-25",
185 "category": "mcp-core",
186 "status": "pass",
187 "durationMs": 12,
188 "assertions": [{"name": "initialize_succeeds", "status": "pass"}]
189 }]"#,
190 )?;
191
192 let scenarios = load_scenarios_from_dir(dir.join("scenarios"))?;
193 let results = load_results_from_dir(dir.join("results"))?;
194
195 assert_eq!(scenarios.len(), 1);
196 assert_eq!(results.len(), 1);
197 assert_eq!(scenarios[0].id, "initialize");
198 assert_eq!(results[0].peer, "js");
199 Ok(())
200 }
201
202 #[test]
203 fn load_scenarios_rejects_missing_directory() -> Result<(), LoadError> {
204 let dir = unique_dir("chio-conformance-load-missing-scenarios")?;
205 let _ = fs::remove_dir_all(&dir);
206
207 match load_scenarios_from_dir(&dir) {
208 Ok(scenarios) => panic!("missing scenario directory should fail: {scenarios:?}"),
209 Err(error) => assert!(error.to_string().contains("directory")),
210 }
211 Ok(())
212 }
213
214 #[test]
215 fn load_scenarios_rejects_empty_directory() -> Result<(), LoadError> {
216 let dir = unique_dir("chio-conformance-load-empty-scenarios")?;
217 let _ = fs::remove_dir_all(&dir);
218 fs::create_dir_all(&dir)?;
219
220 match load_scenarios_from_dir(&dir) {
221 Ok(scenarios) => panic!("empty scenario directory should fail: {scenarios:?}"),
222 Err(error) => assert!(error.to_string().contains("empty")),
223 }
224
225 let _ = fs::remove_dir_all(dir);
226 Ok(())
227 }
228
229 #[test]
230 fn load_results_rejects_missing_directory() -> Result<(), LoadError> {
231 let dir = unique_dir("chio-conformance-load-missing-results")?;
232 let _ = fs::remove_dir_all(&dir);
233
234 match load_results_from_dir(&dir) {
235 Ok(results) => panic!("missing results directory should fail: {results:?}"),
236 Err(error) => assert!(error.to_string().contains("directory")),
237 }
238 Ok(())
239 }
240
241 #[cfg(unix)]
242 #[test]
243 fn collect_json_files_rejects_symlinked_json_file() -> Result<(), LoadError> {
244 let dir = unique_dir("chio-conformance-load-symlink")?;
245 let outside = unique_dir("chio-conformance-load-outside")?;
246 fs::create_dir_all(&dir)?;
247 fs::create_dir_all(&outside)?;
248 fs::write(outside.join("escape.json"), "{}")?;
249 std::os::unix::fs::symlink(outside.join("escape.json"), dir.join("escape.json"))?;
250
251 match collect_json_files(&dir) {
252 Ok(files) => panic!("symlinked JSON should fail closed: {files:?}"),
253 Err(error) => assert!(error.to_string().contains("symlink")),
254 }
255 let _ = fs::remove_dir_all(dir);
256 let _ = fs::remove_dir_all(outside);
257 Ok(())
258 }
259}