#![allow(clippy::disallowed_methods)]
#![cfg(feature = "model-tests")]
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Command;
use aprender::format::layout_contract::{
enforce_embedding_contract, enforce_import_contract, enforce_matmul_contract, LayoutContract,
};
use aprender::format::model_family::{
build_default_registry, Activation, AttentionType, MlpType, NormType, PositionalEncoding,
KNOWN_FAMILIES,
};
use aprender::format::rosetta::FormatType;
use aprender::format::validated_tensors::{RowMajor, ValidatedEmbedding, ValidatedWeight};
use tempfile::NamedTempFile;
fn collect_rs_files(dir: &Path) -> Vec<PathBuf> {
let mut files = Vec::new();
if !dir.exists() || !dir.is_dir() {
return files;
}
if let Ok(entries) = std::fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
let name = path.file_name().unwrap_or_default().to_string_lossy();
if name.starts_with('.') || name == "target" {
continue;
}
files.extend(collect_rs_files(&path));
} else if path.extension().map_or(false, |ext| ext == "rs") {
files.push(path);
}
}
}
files
}
fn project_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")
}
fn crate_dir(crate_name: &str) -> PathBuf {
let dir = project_root().join("crates").join(crate_name);
assert!(
dir.is_dir(),
"crate `{crate_name}` is not at crates/{crate_name}. A crate rename must \
update this suite; it must not silently make a gate vacuous."
);
dir
}
fn crate_src_text(crate_name: &str) -> String {
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
static CACHE: OnceLock<Mutex<HashMap<String, &'static str>>> = OnceLock::new();
let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
let mut guard = cache.lock().expect("crate_src_text cache poisoned");
if let Some(hit) = guard.get(crate_name) {
return (*hit).to_string();
}
let src = crate_dir(crate_name).join("src");
assert!(
src.is_dir(),
"crate `{crate_name}` has no src/ directory at {}",
src.display()
);
let files: Vec<PathBuf> = collect_rs_files(&src)
.into_iter()
.filter(|p| is_scannable_production_source(p))
.collect();
assert!(
!files.is_empty(),
"crate `{crate_name}` src/ contains no .rs files -- refusing to evaluate a \
gate against an empty corpus (a vacuous pass is worse than a failure)"
);
let mut text = String::new();
for path in files {
if let Ok(content) = std::fs::read_to_string(&path) {
text.push_str(&content);
text.push('\n');
}
}
let leaked: &'static str = Box::leak(text.into_boxed_str());
guard.insert(crate_name.to_string(), leaked);
leaked.to_string()
}
fn spec_text() -> String {
let candidates = [
project_root()
.join("docs")
.join("specifications")
.join("archive")
.join("qwen2.5-coder-showcase-demo.md"),
project_root()
.join("docs")
.join("specifications")
.join("qwen2.5-coder-showcase-demo.md"),
];
for path in &candidates {
if let Ok(content) = std::fs::read_to_string(path) {
return content;
}
}
panic!(
"qwen2.5-coder-showcase-demo.md found at none of: {}",
candidates
.iter()
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(", ")
)
}
fn suite_source_text() -> String {
let tests_dir = crate_dir("aprender-core").join("tests");
let mut text = std::fs::read_to_string(tests_dir.join("falsification_spec_v10_tests.rs"))
.expect("suite root readable");
for name in suite_include_names() {
let path = tests_dir.join("includes").join(&name);
text.push_str(
&std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("suite include {} unreadable: {e}", path.display())),
);
text.push('\n');
}
text
}
fn suite_include_names() -> Vec<String> {
let root = crate_dir("aprender-core")
.join("tests")
.join("falsification_spec_v10_tests.rs");
let text = std::fs::read_to_string(&root).expect("suite root readable");
let mut names = Vec::new();
for line in text.lines() {
let trimmed = line.trim();
let Some(rest) = trimmed.strip_prefix(concat!("include", "!(\"includes/")) else {
continue;
};
if let Some(end) = rest.find('"') {
names.push(rest[..end].to_string());
}
}
names
}
fn build_script_text(crate_name: &str) -> String {
let dir = crate_dir(crate_name);
let mut text = std::fs::read_to_string(dir.join("build.rs"))
.unwrap_or_else(|e| panic!("{crate_name}/build.rs unreadable: {e}"));
let includes: Vec<String> = text
.lines()
.filter_map(|l| {
let rest = l.trim().strip_prefix(concat!("include", "!(\""))?;
let end = rest.find('"')?;
Some(rest[..end].to_string())
})
.collect();
for name in includes {
if let Ok(extra) = std::fs::read_to_string(dir.join(&name)) {
text.push('\n');
text.push_str(&extra);
}
}
text
}
fn is_scannable_production_source(path: &Path) -> bool {
let s = path.to_string_lossy().replace('\\', "/");
if s.contains("/tests/") || s.contains("/examples/") || s.contains("/benches/") {
return false;
}
let name = path
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_default();
!(name == "tests.rs" || name.starts_with("tests_") || name.contains("_tests"))
}
fn production_rs_files() -> Vec<PathBuf> {
let mut files = Vec::new();
for dir in [project_root().join("src"), project_root().join("crates")] {
files.extend(
collect_rs_files(&dir)
.into_iter()
.filter(|p| is_scannable_production_source(p)),
);
}
assert!(
!files.is_empty(),
"production source scan found 0 files -- a scan over an empty universe \
passes every assertion put to it"
);
files
}
fn strip_trailing_comment(line: &str) -> &str {
match line.find("//") {
Some(pos) => &line[..pos],
None => line,
}
}
fn contains_f32_token(haystack: &str) -> bool {
let bytes = haystack.as_bytes();
let mut from = 0usize;
while let Some(rel) = haystack[from..].find("F32") {
let start = from + rel;
let end = start + 3;
let before_ok =
start == 0 || !(bytes[start - 1].is_ascii_alphanumeric() || bytes[start - 1] == b'_');
let after_ok =
end >= bytes.len() || !(bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_');
if before_ok && after_ok {
return true;
}
from = end;
}
false
}
fn search_dir_for_file(search_root: &Path, filename: &str) -> Option<PathBuf> {
let mut dirs_to_visit = vec![search_root.to_path_buf()];
while let Some(dir) = dirs_to_visit.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
dirs_to_visit.push(path);
} else if path.file_name().map_or(false, |n| n == filename) {
return Some(path);
}
}
}
None
}
fn target_dir_candidates() -> Vec<PathBuf> {
let mut roots = Vec::new();
if let Ok(exe) = std::env::current_exe() {
if let Some(target) = exe
.parent()
.and_then(|p| p.parent())
.and_then(|p| p.parent())
{
roots.push(target.to_path_buf());
}
}
if let Ok(dir) = std::env::var("CARGO_TARGET_DIR") {
roots.push(PathBuf::from(dir));
}
roots.push(project_root().join("target"));
roots
}
fn find_generated_file(filename: &str) -> Option<PathBuf> {
for target_dir in target_dir_candidates() {
for profile in &["debug", "release"] {
let search_root = target_dir.join(profile).join("build");
if !search_root.exists() {
continue;
}
if let Some(found) = search_dir_for_file(&search_root, filename) {
return Some(found);
}
}
}
None
}
fn model_dir() -> PathBuf {
if let Ok(dir) = std::env::var("MODEL_DIR") {
PathBuf::from(dir)
} else {
project_root().join("models")
}
}
fn gguf_model_path() -> Option<PathBuf> {
let path = model_dir().join("qwen2.5-coder-0.5b-instruct-q4_k_m.gguf");
if path.exists() {
Some(path)
} else {
None
}
}
fn apr_model_path() -> Option<PathBuf> {
let path = model_dir().join("qwen2.5-coder-0.5b-instruct-q4_k_m.apr");
if !path.exists() {
return None;
}
let bin = apr_binary();
let output = Command::new(&bin)
.args(["tensors", path.to_str().unwrap()])
.output()
.ok()?;
if output.status.success() {
Some(path)
} else {
eprintln!(
"SKIP: APR model exists but is corrupt/incompatible: {}",
String::from_utf8_lossy(&output.stderr)
);
None
}
}
fn safetensors_model_dir() -> Option<PathBuf> {
let candidates: Vec<PathBuf> = [
std::env::var("MODEL_DIR")
.ok()
.map(|d| PathBuf::from(d).join("qwen2.5-coder-0.5b-instruct")),
std::env::var("HOME")
.ok()
.map(|h| PathBuf::from(h).join("models/qwen2.5-coder-0.5b-instruct")),
]
.into_iter()
.flatten()
.collect();
candidates
.into_iter()
.find(|path| path.join("model.safetensors").exists())
}
fn apr_binary() -> PathBuf {
let target_bases = target_dir_candidates();
for base in &target_bases {
let release = base.join("release").join("apr");
if release.exists() {
return release;
}
let debug = base.join("debug").join("apr");
if debug.exists() {
return debug;
}
}
PathBuf::from("apr")
}
fn which_ollama() -> Option<PathBuf> {
let output = Command::new("which").arg("ollama").output().ok()?;
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
if path.is_empty() {
None
} else {
Some(PathBuf::from(path))
}
} else {
None
}
}
fn run_apr(args: &[&str]) -> (bool, String, String) {
let bin = apr_binary();
let output = Command::new(&bin)
.args(args)
.current_dir(project_root())
.output()
.unwrap_or_else(|e| panic!("Failed to run apr at {}: {}", bin.display(), e));
(
output.status.success(),
String::from_utf8_lossy(&output.stdout).to_string(),
String::from_utf8_lossy(&output.stderr).to_string(),
)
}
macro_rules! require_model {
($path_opt:expr, $name:expr) => {
match $path_opt {
Some(p) => p,
None => {
eprintln!(
"SKIP[fixture]: {} not found. Set MODEL_DIR or download with `apr pull`",
$name
);
return;
}
}
};
}
const SKIP_SITE_BASELINE: usize = 62;
const SKIP_SITE_SLACK: usize = 5;
fn count_skip_sites(text: &str) -> (usize, usize) {
let fixture = text.matches(concat!("require_model", "!(")).count();
let mut early = 0usize;
let mut saw_test_attr = false;
let mut in_test = false;
for line in text.lines() {
let trimmed = line.trim();
if in_test {
if line == "}" {
in_test = false;
saw_test_attr = false;
} else if !trimmed.starts_with("//") && trimmed.contains("return;") {
early += 1;
}
continue;
}
if trimmed.starts_with("#[test]") {
saw_test_attr = true;
} else if saw_test_attr && trimmed.starts_with("fn ") {
in_test = true;
} else if saw_test_attr && !trimmed.is_empty() && !trimmed.starts_with('#') {
saw_test_attr = false;
}
}
(fixture, early)
}
#[test]
fn f_meta_001_every_include_is_included() {
let includes_dir = crate_dir("aprender-core").join("tests").join("includes");
let declared: std::collections::HashSet<String> = suite_include_names().into_iter().collect();
assert!(
!declared.is_empty(),
"F-META-001: the suite root declares no include!() fragments"
);
let mut orphans = Vec::new();
for name in &declared {
let path = includes_dir.join(name);
assert!(
path.is_file(),
"F-META-001: suite includes `{name}`, which does not exist at {}",
path.display()
);
}
for entry in std::fs::read_dir(&includes_dir)
.expect("includes/ readable")
.flatten()
{
let name = entry.file_name().to_string_lossy().to_string();
let is_ours = name.starts_with("falsification_spec_v10_");
if is_ours && !declared.contains(&name) {
orphans.push(name);
}
}
assert!(
orphans.is_empty(),
"F-META-001: falsification_spec_v10 fragments that no include!() reaches: {}",
orphans.join(", ")
);
}
#[test]
fn f_meta_002_skip_class_is_bounded() {
let tests_dir = crate_dir("aprender-core").join("tests");
let mut fixture = 0usize;
let mut early = 0usize;
let names = suite_include_names();
assert!(
!names.is_empty(),
"F-META-002: the suite declares no fragments -- an empty universe is \
not a measurement"
);
for name in names {
let text = std::fs::read_to_string(tests_dir.join("includes").join(&name))
.unwrap_or_else(|e| panic!("suite include {name} unreadable: {e}"));
let (f, e) = count_skip_sites(&text);
fixture += f;
early += e;
}
let sites = fixture + early;
assert!(
fixture > 0 && early > 0,
"F-META-002: counted {fixture} fixture gates and {early} other early \
returns. A zero on either half means count_skip_sites stopped matching, \
not that the skips went away."
);
assert!(
sites <= SKIP_SITE_BASELINE,
"F-META-002: {sites} silent-skip sites ({fixture} `require_model!` + \
{early} other early returns inside #[test]) against a baseline of \
{SKIP_SITE_BASELINE}. A gate that returns before asserting reports `ok` \
while proving nothing -- convert it to a real assertion instead of \
raising the baseline."
);
assert!(
sites + SKIP_SITE_SLACK >= SKIP_SITE_BASELINE,
"F-META-002: silent-skip sites fell to {sites} ({fixture} fixture + \
{early} other). Lower SKIP_SITE_BASELINE to {sites} so the gain is \
locked in."
);
}
fn fold_shell_continuations(text: &str) -> Vec<String> {
let mut out = Vec::new();
let mut buf = String::new();
for raw in text.lines() {
let piece = if buf.is_empty() {
raw.to_string()
} else {
format!("{} {}", buf, raw.trim_start())
};
match piece.strip_suffix('\\') {
Some(head) => buf = head.to_string(),
None => {
buf.clear();
out.push(piece);
}
}
}
if !buf.is_empty() {
out.push(buf);
}
out
}
#[test]
fn f_meta_003_suite_is_named_by_a_workflow() {
let workflows = project_root().join(".github").join("workflows");
let mut wired = false;
let mut scanned = 0usize;
for entry in std::fs::read_dir(&workflows)
.expect(".github/workflows readable")
.flatten()
{
let path = entry.path();
if path.extension().map_or(true, |e| e != "yml" && e != "yaml") {
continue;
}
scanned += 1;
let Ok(text) = std::fs::read_to_string(&path) else {
continue;
};
for line in fold_shell_continuations(&text) {
let code = match line.find('#') {
Some(pos) => line[..pos].to_string(),
None => line,
};
if code.contains("--test falsification_spec_v10_tests") && code.contains("--features") {
wired = true;
}
}
}
assert!(
scanned > 5,
"F-META-003: scanned only {scanned} workflow files -- the scan is broken, \
not the wiring"
);
assert!(
wired,
"F-META-003: no workflow runs `--test falsification_spec_v10_tests` with a \
`--features` flag. This suite is dark again (aprender#2522)."
);
}
include!("includes/falsification_spec_v10_ground_truth.rs");
include!("includes/falsification_spec_v10_cli_interface.rs");
include!("includes/falsification_spec_v10_model_spec.rs");
include!("includes/falsification_spec_v10_checklist.rs");
include!("includes/f_ollama_00.rs");
include!("includes/falsification_spec_v10_definition_of_done.rs");
include!("includes/falsification_spec_v10_ml_diagnostics.rs");
include!("includes/f_trueno_00.rs");
include!("includes/f_realize_0.rs");
include!("includes/falsification_spec_v10_contract_model.rs");
include!("includes/falsification_spec_v10_qwen2_7b_params.rs");