#![forbid(unsafe_code)]
#![cfg_attr(
test,
allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::doc_markdown
)
)]
use chrono::{DateTime, Utc};
use std::fmt;
use thiserror::Error;
#[derive(Error, Debug)]
pub enum CertifyError {
#[error("CSV parse error at line {line}: {message}")]
CsvParse {
line: usize,
message: String,
},
#[error("README marker not found: {0}")]
MarkerNotFound(String),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
}
pub type Result<T> = std::result::Result<T, CertifyError>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum CertificationStatus {
Certified,
Provisional,
Blocked,
#[default]
Pending,
}
impl CertificationStatus {
#[must_use]
pub fn parse(s: &str) -> Option<Self> {
match s.to_uppercase().as_str() {
"CERTIFIED" => Some(Self::Certified),
"PROVISIONAL" => Some(Self::Provisional),
"BLOCKED" => Some(Self::Blocked),
"PENDING" => Some(Self::Pending),
_ => None,
}
}
#[must_use]
pub const fn badge(&self) -> &'static str {
match self {
Self::Certified => "",
Self::Provisional => "",
Self::Blocked => "",
Self::Pending => "",
}
}
}
impl fmt::Display for CertificationStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Certified => write!(f, "CERTIFIED"),
Self::Provisional => write!(f, "PROVISIONAL"),
Self::Blocked => write!(f, "BLOCKED"),
Self::Pending => write!(f, "PENDING"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SizeCategory {
Tiny,
#[default]
Small,
Medium,
Large,
XLarge,
}
impl SizeCategory {
#[must_use]
pub fn parse(s: &str) -> Option<Self> {
match s.to_lowercase().as_str() {
"tiny" => Some(Self::Tiny),
"small" => Some(Self::Small),
"medium" => Some(Self::Medium),
"large" => Some(Self::Large),
"xlarge" => Some(Self::XLarge),
_ => None,
}
}
}
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone)]
pub struct ModelCertification {
pub model_id: String,
pub family: String,
pub parameters: String,
pub size_category: SizeCategory,
pub status: CertificationStatus,
pub mqs_score: u32,
pub grade: String,
pub certified_tier: String,
pub last_certified: Option<DateTime<Utc>>,
pub g1: bool,
pub g2: bool,
pub g3: bool,
pub g4: bool,
pub tps_gguf_cpu: Option<f64>,
pub tps_gguf_gpu: Option<f64>,
pub tps_apr_cpu: Option<f64>,
pub tps_apr_gpu: Option<f64>,
pub tps_st_cpu: Option<f64>,
pub tps_st_gpu: Option<f64>,
pub provenance_verified: bool,
pub kernel_proof_ref: Option<String>,
}
impl ModelCertification {
#[must_use]
pub fn short_name(&self) -> &str {
self.model_id
.split('/')
.next_back()
.unwrap_or(&self.model_id)
}
#[must_use]
pub fn hf_url(&self) -> String {
format!("https://huggingface.co/{}", self.model_id)
}
#[must_use]
pub fn markdown_link(&self) -> String {
format!("[{}]({})", self.short_name(), self.hf_url())
}
#[must_use]
pub const fn gateway_symbol(passed: bool, status: CertificationStatus) -> &'static str {
if matches!(status, CertificationStatus::Pending) {
"-"
} else if passed {
"\u{2713}" } else {
"\u{2717}" }
}
#[must_use]
pub fn param_count(&self) -> f64 {
self.parameters
.trim_end_matches('B')
.parse::<f64>()
.unwrap_or(0.0)
}
}
const REQUIRED_CSV_COLUMNS: &[&str] = &[
"model_id",
"family",
"parameters",
"size_category",
"status",
"mqs_score",
"grade",
"certified_tier",
"last_certified",
"g1",
"g2",
"g3",
"g4",
];
#[allow(clippy::similar_names)]
pub fn parse_csv(content: &str) -> Result<Vec<ModelCertification>> {
let mut models = Vec::new();
let mut lines = content.lines().enumerate();
let Some((_, header)) = lines.next() else {
return Ok(models);
};
let header_fields = csv_split(header);
let col: std::collections::HashMap<&str, usize> = header_fields
.iter()
.enumerate()
.map(|(i, name)| (name.as_str(), i))
.collect();
for &name in REQUIRED_CSV_COLUMNS {
if !col.contains_key(name) {
return Err(CertifyError::CsvParse {
line: 1,
message: format!("missing required column '{name}'"),
});
}
}
for (line_num, line) in lines {
if line.trim().is_empty() {
continue;
}
let fields = csv_split(line);
let get = |name: &str| -> &str {
col.get(name)
.and_then(|&i| fields.get(i).map(String::as_str))
.unwrap_or("")
};
let get_opt = |name: &str| -> Option<String> {
col.get(name)
.and_then(|&i| fields.get(i))
.filter(|s| !s.is_empty())
.cloned()
};
let last_certified = DateTime::parse_from_rfc3339(get("last_certified"))
.ok()
.map(|dt| dt.with_timezone(&Utc));
models.push(ModelCertification {
model_id: get("model_id").to_owned(),
family: get("family").to_owned(),
parameters: get("parameters").to_owned(),
size_category: SizeCategory::parse(get("size_category")).ok_or_else(|| {
CertifyError::CsvParse {
line: line_num + 1,
message: format!("invalid size_category: '{}'", get("size_category")),
}
})?,
status: CertificationStatus::parse(get("status")).ok_or_else(|| {
CertifyError::CsvParse {
line: line_num + 1,
message: format!("invalid status: '{}'", get("status")),
}
})?,
mqs_score: get("mqs_score")
.parse()
.map_err(|_| CertifyError::CsvParse {
line: line_num + 1,
message: format!("invalid mqs_score: '{}'", get("mqs_score")),
})?,
grade: get("grade").to_owned(),
certified_tier: get("certified_tier").to_owned(),
last_certified,
g1: get("g1").to_lowercase() == "true",
g2: get("g2").to_lowercase() == "true",
g3: get("g3").to_lowercase() == "true",
g4: get("g4").to_lowercase() == "true",
tps_gguf_cpu: get_opt("tps_gguf_cpu").and_then(|s| s.parse().ok()),
tps_gguf_gpu: get_opt("tps_gguf_gpu").and_then(|s| s.parse().ok()),
tps_apr_cpu: get_opt("tps_apr_cpu").and_then(|s| s.parse().ok()),
tps_apr_gpu: get_opt("tps_apr_gpu").and_then(|s| s.parse().ok()),
tps_st_cpu: get_opt("tps_st_cpu").and_then(|s| s.parse().ok()),
tps_st_gpu: get_opt("tps_st_gpu").and_then(|s| s.parse().ok()),
provenance_verified: get_opt("provenance_verified")
.is_some_and(|s| s.to_lowercase() == "true"),
kernel_proof_ref: get_opt("kernel_proof_ref"),
});
}
Ok(models)
}
#[must_use]
pub fn generate_summary(models: &[ModelCertification], timestamp: &str) -> String {
let total = models.len();
let certified = models
.iter()
.filter(|m| matches!(m.status, CertificationStatus::Certified))
.count();
let provisional = models
.iter()
.filter(|m| matches!(m.status, CertificationStatus::Provisional))
.count();
let blocked = models
.iter()
.filter(|m| matches!(m.status, CertificationStatus::Blocked))
.count();
let pending = models
.iter()
.filter(|m| matches!(m.status, CertificationStatus::Pending))
.count();
format!(
r"**Certification Summary** (updated: {timestamp})
| Status | Count |
|--------|-------|
| Certified | {certified}/{total} |
| Provisional | {provisional}/{total} |
| Blocked | {blocked}/{total} |
| Pending | {pending}/{total} |
**Priority Family:** Qwen Coder (see [Certified Testing Spec](docs/specifications/certified-testing.md))"
)
}
#[must_use]
pub fn generate_table(models: &[ModelCertification]) -> String {
let mut lines = Vec::new();
lines.push(
"| Model | Family | Size | Status | MQS | Grade | G1-4 | Prov | GGUF CPU | GGUF GPU | APR CPU | APR GPU | ST CPU | ST GPU |"
.to_string(),
);
lines.push(
"|-------|--------|------|--------|-----|-------|------|------|----------|----------|---------|---------|--------|--------|"
.to_string(),
);
let mut sorted: Vec<_> = models.iter().collect();
sorted.sort_by(|a, b| {
a.family.cmp(&b.family).then_with(|| {
a.param_count()
.partial_cmp(&b.param_count())
.unwrap_or(std::cmp::Ordering::Equal)
})
});
for m in sorted {
let gateways = if matches!(m.status, CertificationStatus::Pending) {
"-".to_string()
} else if m.g1 && m.g2 && m.g3 && m.g4 {
"\u{2713}".to_string() } else {
"\u{2717}".to_string() };
let prov = if matches!(m.status, CertificationStatus::Pending) {
"-"
} else if m.provenance_verified {
"\u{2713}" } else {
"\u{2717}" };
let fmt = |v: Option<f64>| v.map_or_else(|| "-".to_string(), |x| format!("{x:.1}"));
lines.push(format!(
"| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |",
m.markdown_link(),
m.family,
m.parameters,
m.status.badge(),
m.mqs_score,
m.grade,
gateways,
prov,
fmt(m.tps_gguf_cpu),
fmt(m.tps_gguf_gpu),
fmt(m.tps_apr_cpu),
fmt(m.tps_apr_gpu),
fmt(m.tps_st_cpu),
fmt(m.tps_st_gpu),
));
}
lines.join("\n")
}
pub const START_MARKER: &str = "<!-- CERTIFICATION_TABLE_START -->";
pub const END_MARKER: &str = "<!-- CERTIFICATION_TABLE_END -->";
#[must_use]
pub fn write_csv(models: &[ModelCertification]) -> String {
let mut lines = Vec::new();
lines.push(
"model_id,family,parameters,size_category,status,mqs_score,grade,certified_tier,last_certified,g1,g2,g3,g4,tps_gguf_cpu,tps_gguf_gpu,tps_apr_cpu,tps_apr_gpu,tps_st_cpu,tps_st_gpu,provenance_verified,kernel_proof_ref"
.to_string(),
);
for m in models {
let size_cat = match m.size_category {
SizeCategory::Tiny => "tiny",
SizeCategory::Small => "small",
SizeCategory::Medium => "medium",
SizeCategory::Large => "large",
SizeCategory::XLarge => "xlarge",
};
let last_cert = m
.last_certified
.map_or_else(String::new, |dt| dt.to_rfc3339());
let fmt = |v: Option<f64>| v.map_or(String::new(), |x| format!("{x:.1}"));
lines.push(format!(
"{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}",
csv_quote(&m.model_id),
csv_quote(&m.family),
csv_quote(&m.parameters),
size_cat,
m.status,
m.mqs_score,
csv_quote(&m.grade),
csv_quote(&m.certified_tier),
last_cert,
m.g1,
m.g2,
m.g3,
m.g4,
fmt(m.tps_gguf_cpu),
fmt(m.tps_gguf_gpu),
fmt(m.tps_apr_cpu),
fmt(m.tps_apr_gpu),
fmt(m.tps_st_cpu),
fmt(m.tps_st_gpu),
m.provenance_verified,
csv_quote(m.kernel_proof_ref.as_deref().unwrap_or("")),
));
}
lines.join("\n") + "\n"
}
fn csv_quote(field: &str) -> String {
if field.contains(',') || field.contains('"') || field.contains('\n') || field.contains('\r') {
format!("\"{}\"", field.replace('"', "\"\""))
} else {
field.to_string()
}
}
fn csv_split(line: &str) -> Vec<String> {
let mut fields = Vec::new();
let mut current = String::new();
let mut in_quotes = false;
let mut chars = line.chars().peekable();
while let Some(ch) = chars.next() {
match ch {
'"' if in_quotes => {
if chars.peek() == Some(&'"') {
current.push('"');
chars.next();
} else {
in_quotes = false;
}
}
'"' if !in_quotes && current.is_empty() => {
in_quotes = true;
}
',' if !in_quotes => {
fields.push(std::mem::take(&mut current));
}
_ => {
current.push(ch);
}
}
}
fields.push(current);
fields
}
include!("lib_scoring.rs");
#[cfg(test)]
#[path = "lib_tests.rs"]
mod tests;