use std::collections::{BTreeSet, HashMap};
use std::fs;
use std::path::{Path, PathBuf};
use serde_json::{json, Value};
use super::{build_corpus, cases, selected, Case, CaseResult};
use crate::digest::hash_hex;
use crate::json::{pretty, py_str};
use crate::pyval::py_path;
use crate::{codes, Error, Result, FORMAT_VERSION, VERSION};
pub const SCHEMA: &str = "medh5-sample-1.0.schema.json";
pub const CLINICAL_SCHEMA: &str = crate::clinical::schema::SCHEMA_FILE;
pub const COMPANION_SCHEMAS: [&str; 2] = [crate::companion::task::SCHEMA_FILE, crate::companion::cache::SCHEMA_FILE];
pub const CHECKSUMS: &str = "SHA256SUMS";
pub fn publish(outdir: &Path, names: Option<&[String]>) -> Result<PathBuf> {
build_corpus(outdir, names)?;
let chosen: Vec<&Case> = selected(names);
let table = json!({
"format": FORMAT_VERSION,
"codes": codes::all()
.iter()
.map(|c| json!({"code": c.code, "severity": c.severity, "domain": c.domain, "summary": c.summary}))
.collect::<Vec<Value>>(),
});
fs::write(outdir.join("codes.json"), pretty(&table) + "\n")?;
fs::write(outdir.join(SCHEMA), crate::document::schema_text())?;
fs::write(outdir.join(CLINICAL_SCHEMA), crate::clinical::schema::schema_text())?;
fs::write(outdir.join(COMPANION_SCHEMAS[0]), crate::companion::task::schema_text())?;
fs::write(outdir.join(COMPANION_SCHEMAS[1]), crate::companion::cache::schema_text())?;
if super::companion::SOURCES.iter().all(|f| outdir.join(f).exists()) {
super::companion::write_fixtures(outdir)?;
}
fs::write(outdir.join("README.md"), readme(&chosen))?;
write_checksums(outdir)?;
Ok(outdir.to_path_buf())
}
fn published_files(root: &Path) -> Result<Vec<String>> {
fn walk(dir: &Path, prefix: &[String], out: &mut Vec<Vec<String>>) -> Result<()> {
for entry in fs::read_dir(dir)? {
let entry = entry?;
let mut parts = prefix.to_vec();
parts.push(entry.file_name().to_string_lossy().into_owned());
let kind = entry.file_type()?;
if kind.is_dir() {
walk(&entry.path(), &parts, out)?;
} else if entry.path().is_file() {
out.push(parts);
}
}
Ok(())
}
let mut found = Vec::new();
walk(root, &[], &mut found)?;
found.sort();
Ok(found.into_iter().map(|parts| parts.join("/")).collect())
}
fn write_checksums(root: &Path) -> Result<PathBuf> {
let mut lines = Vec::new();
for name in published_files(root)? {
if name.rsplit('/').next() == Some(CHECKSUMS) {
continue;
}
let digest = hash_hex("sha256", &fs::read(root.join(&name))?)?;
lines.push(format!("{digest} {name}"));
}
let target = root.join(CHECKSUMS);
fs::write(&target, lines.join("\n") + "\n")?;
Ok(target)
}
pub fn check_checksums(root: &Path) -> Result<Vec<String>> {
let listing = root.join(CHECKSUMS);
let text = fs::read_to_string(&listing).map_err(|e| crate::error::os_error(&e, py_path(&listing)))?;
let mut bad = Vec::new();
for line in text.lines() {
if line.trim().is_empty() {
continue;
}
let (digest, name) = line.split_once(" ").unwrap_or((line, ""));
let path = root.join(name);
let matches = path.exists()
&& hash_hex("sha256", &fs::read(&path).map_err(|e| crate::error::os_error(&e, py_path(&path)))?)? == digest;
if !matches {
bad.push(name.to_string());
}
}
Ok(bad)
}
pub fn load_manifest(root: &Path) -> Result<Value> {
let path = root.join("expected.json");
if !path.exists() {
return Err(Error::invalid(format!(
"{} not found --- build the suite first with `medh5 conformance publish`",
py_path(&path)
)));
}
let text = fs::read_to_string(&path).map_err(|e| crate::error::os_error(&e, py_path(&path)))?;
crate::json::loads(&text).map_err(|e| Error::Value(format!("not valid JSON: {e}")))
}
fn submitted_codes(entry: &Value) -> (BTreeSet<String>, BTreeSet<String>) {
let mut errors = BTreeSet::new();
let mut warnings = BTreeSet::new();
if let Some(diagnostics) = entry.get("diagnostics") {
for diagnostic in diagnostics.as_array().into_iter().flatten() {
let code = diagnostic.get("code").map(py_str).unwrap_or_default();
if code.is_empty() {
continue;
}
let severity = diagnostic.get("severity").map(py_str).unwrap_or_else(|| "error".into());
if severity == "warning" {
warnings.insert(code);
} else {
errors.insert(code);
}
}
return (errors, warnings);
}
let list = |key: &str| -> BTreeSet<String> {
entry.get(key).and_then(Value::as_array).map(|a| a.iter().map(py_str).collect()).unwrap_or_default()
};
(list("errors"), list("warnings"))
}
fn submitted_name(entry: &Value) -> String {
let raw = ["file", "path", "name"]
.iter()
.filter_map(|k| entry.get(*k))
.find(|v| crate::json::py_truthy(v))
.map(py_str)
.unwrap_or_default();
Path::new(&raw).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()
}
fn strings_of(record: &Value, key: &str) -> Vec<String> {
record.get(key).and_then(Value::as_array).map(|a| a.iter().map(py_str).collect()).unwrap_or_default()
}
pub fn score(root: &Path, submitted: &[Value]) -> Result<Vec<CaseResult>> {
let manifest = load_manifest(root)?;
let by_file: HashMap<String, &Value> = submitted.iter().map(|e| (submitted_name(e), e)).collect();
let mut results = Vec::new();
for record in manifest.get("cases").and_then(Value::as_array).into_iter().flatten() {
let name = record.get("name").map(py_str).unwrap_or_default();
let case = cases().iter().find(|c| c.name == name).cloned().unwrap_or_else(|| Case::from_record(record));
let file = record.get("file").map(py_str).unwrap_or_default();
let mut result = CaseResult::new(case, py_path(&root.join(&file)));
let Some(entry) = by_file.get(&file) else {
result.error = Some("no result submitted for this case".into());
results.push(result);
continue;
};
let (errors, warnings) = submitted_codes(entry);
let (missing, unexpected) = super::differences(
&strings_of(record, "expect_errors"),
&strings_of(record, "expect_warnings"),
&errors,
&warnings,
);
result.got_errors = errors.into_iter().collect();
result.got_warnings = warnings.into_iter().collect();
result.missing = missing;
result.unexpected = unexpected;
results.push(result);
}
Ok(results)
}
pub fn summarize(results: &[CaseResult]) -> Value {
let failures: Vec<&CaseResult> = results.iter().filter(|r| !r.ok()).collect();
json!({
"cases": results.len(),
"passed": results.len() - failures.len(),
"failed": failures.len(),
"ok": failures.is_empty(),
"failures": failures.iter().map(|r| r.to_json()).collect::<Vec<Value>>(),
})
}
fn readme(cases: &[&Case]) -> String {
let valid = cases.iter().filter(|c| c.valid()).count();
let shards = cases.iter().filter(|c| c.suffix == ".medh5c").count();
let total = cases.len();
let kinds = format!("the cases: {} samples and {shards} collections", total - shards);
let covered: BTreeSet<&str> =
cases.iter().flat_map(|c| c.errors.iter().chain(&c.warnings)).map(String::as_str).collect();
let invalid = total - valid;
let n_covered = covered.len();
let n_codes = codes::all().len();
let (task_schema, cache_schema) = (COMPANION_SCHEMAS[0], COMPANION_SCHEMAS[1]);
format!(
r#"# MEDH5 {FORMAT_VERSION} conformance suite
Generated by medh5 {VERSION}. {total} cases: {valid} valid files a
conforming implementation must accept, {invalid} invalid ones it must
reject with specific diagnostic codes. {n_covered} of the {n_codes} codes in
the specification's tables (1.0 §15.2 and 1.1 §11.2) appear here.
## What is in this directory
| File | What it is |
|---|---|
| `*.medh5`, `*.medh5c` | {kinds} |
| `expected.json` | per case, the exact codes a conforming validator must emit |
| `codes.json` | the diagnostic code table as data (1.0 §15.2 and 1.1 §11.2) |
| `{SCHEMA}` | the JSON Schema for the `/meta` document |
| `{CLINICAL_SCHEMA}` | the JSON Schema for `clinical/meta` and clinical records (format 1.1) |
| `{task_schema}`, `{cache_schema}` | the task-and-cache contract's schemas (not part of the corpus) |
| `{CHECKSUMS}` | sha256 of every file above |
## Running it
Validate every case **at the level its manifest entry declares**, and write one
JSON array:
```json
[
{{"file": "core-minimal.medh5", "errors": [], "warnings": []}},
{{"file": "E102-non-orthonormal.medh5", "errors": ["E102"], "warnings": []}}
]
```
Then score it:
```
medh5 conformance score . results.json
```
The scorer also takes the shape `medh5 validate --json` reports a file in: a
`diagnostics` list, each entry with its `code` and `severity`, in place of
`errors` and `warnings` (and `path` in place of `file`). So the reference
implementation is scored through exactly the same door as everybody else:
```python
import json, subprocess
manifest = json.load(open("expected.json"))
results = []
for case in manifest["cases"]:
out = subprocess.run(
["medh5", "validate", case["file"], "--level", case["level"], "--json"],
capture_output=True, text=True,
).stdout
report = json.loads(out)[0]
results.append({{"file": case["file"], "diagnostics": report["diagnostics"]}})
json.dump(results, open("results.json", "w"))
```
## How it is scored
For each case, the errors you report must **equal** the expected errors, and
the warnings the expected warnings: a missing code is a defect you failed to
catch, an extra code is a valid file you rejected. Both fail. Severity is
scored: an expected error you report as a warning is missing as an error, and
a file you report only warnings for is a file you accepted. A case you report
nothing about fails too --- silence about a file you were handed is not a pass.
Diagnostic *messages* are yours to write; only the codes are normative.
## Three things worth knowing before you start
**Validate at the declared `level`, not deeper.** `structural` < `semantic` <
`integrity`. Shallower misses the defect the case exists to test. *Deeper is
not safe either*: most invalid cases were made by editing a valid file, so
their stored digests cover the pre-edit bytes and an integrity pass adds a
`content_id` mismatch the case never claimed. Those cases are marked
`"mutated": true`, and the mismatch is an artifact of how they are built, not
something to diagnose.
**A `.medh5c` case is a collection** (spec §2.2) --- it contains samples rather
than being one. `"file_suffix"` in the manifest tells you which.
**Verify the bytes first.** `{CHECKSUMS}` covers every published file, and
`medh5 conformance score` fails the score when one has drifted, whatever the
cases say, naming it (under `changed` with `--json`). A score over files that
are not the published files is not a score.
The specification is `docs/spec/medh5-1.0.md` with `docs/spec/medh5-1.1.md`
in the medh5 repository, and every case names the clause it tests in
`expected.json`.
## The task-and-cache fixtures
`companion/` holds task manifests (`medh5.task/1`) over this suite's own
clinical samples, each pinned to the content those samples have here, and
`companion/expected.json`: for each, the level it is checked at ---
`validate` (the manifest alone, no file opened) or `preflight` (its sources
opened too) --- the exact set of `T` codes checking it must find, and for a
valid task each row's status, selected event versions, slot images and target
label. They score an implementation of the separately versioned task-and-cache
contract (`docs/spec/task-cache-1.md`), not the format. Resolve each source
`uri` against the manifest's directory. With the reference implementation:
```
medh5 task validate companion/<fixture>.task.json --json
medh5 task preflight companion/<fixture>.task.json --json
```
"#
)
}