1#![forbid(unsafe_code)]
9#![cfg_attr(
10 test,
11 allow(
12 clippy::unwrap_used,
13 clippy::expect_used,
14 clippy::panic,
15 clippy::doc_markdown
16 )
17)]
18
19use chrono::{DateTime, Utc};
20use std::fmt;
21use thiserror::Error;
22
23#[derive(Error, Debug)]
25pub enum CertifyError {
26 #[error("CSV parse error at line {line}: {message}")]
28 CsvParse {
29 line: usize,
31 message: String,
33 },
34
35 #[error("README marker not found: {0}")]
37 MarkerNotFound(String),
38
39 #[error("IO error: {0}")]
41 Io(#[from] std::io::Error),
42}
43
44pub type Result<T> = std::result::Result<T, CertifyError>;
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
49pub enum CertificationStatus {
50 Certified,
52 Provisional,
54 Blocked,
56 #[default]
58 Pending,
59}
60
61impl CertificationStatus {
62 #[must_use]
65 pub fn parse(s: &str) -> Option<Self> {
66 match s.to_uppercase().as_str() {
67 "CERTIFIED" => Some(Self::Certified),
68 "PROVISIONAL" => Some(Self::Provisional),
69 "BLOCKED" => Some(Self::Blocked),
70 "PENDING" => Some(Self::Pending),
71 _ => None,
72 }
73 }
74
75 #[must_use]
77 pub const fn badge(&self) -> &'static str {
78 match self {
79 Self::Certified => "",
80 Self::Provisional => "",
81 Self::Blocked => "",
82 Self::Pending => "",
83 }
84 }
85}
86
87impl fmt::Display for CertificationStatus {
88 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89 match self {
90 Self::Certified => write!(f, "CERTIFIED"),
91 Self::Provisional => write!(f, "PROVISIONAL"),
92 Self::Blocked => write!(f, "BLOCKED"),
93 Self::Pending => write!(f, "PENDING"),
94 }
95 }
96}
97
98#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
100pub enum SizeCategory {
101 Tiny,
103 #[default]
105 Small,
106 Medium,
108 Large,
110 XLarge,
112}
113
114impl SizeCategory {
115 #[must_use]
118 pub fn parse(s: &str) -> Option<Self> {
119 match s.to_lowercase().as_str() {
120 "tiny" => Some(Self::Tiny),
121 "small" => Some(Self::Small),
122 "medium" => Some(Self::Medium),
123 "large" => Some(Self::Large),
124 "xlarge" => Some(Self::XLarge),
125 _ => None,
126 }
127 }
128}
129
130#[allow(clippy::struct_excessive_bools)]
136#[derive(Debug, Clone)]
137pub struct ModelCertification {
138 pub model_id: String,
140 pub family: String,
142 pub parameters: String,
144 pub size_category: SizeCategory,
146 pub status: CertificationStatus,
148 pub mqs_score: u32,
150 pub grade: String,
152 pub certified_tier: String,
154 pub last_certified: Option<DateTime<Utc>>,
156 pub g1: bool,
158 pub g2: bool,
160 pub g3: bool,
162 pub g4: bool,
164 pub tps_gguf_cpu: Option<f64>,
166 pub tps_gguf_gpu: Option<f64>,
168 pub tps_apr_cpu: Option<f64>,
170 pub tps_apr_gpu: Option<f64>,
172 pub tps_st_cpu: Option<f64>,
174 pub tps_st_gpu: Option<f64>,
176 pub provenance_verified: bool,
178 pub kernel_proof_ref: Option<String>,
180}
181
182impl ModelCertification {
183 #[must_use]
185 pub fn short_name(&self) -> &str {
186 self.model_id
187 .split('/')
188 .next_back()
189 .unwrap_or(&self.model_id)
190 }
191
192 #[must_use]
194 pub fn hf_url(&self) -> String {
195 format!("https://huggingface.co/{}", self.model_id)
196 }
197
198 #[must_use]
200 pub fn markdown_link(&self) -> String {
201 format!("[{}]({})", self.short_name(), self.hf_url())
202 }
203
204 #[must_use]
206 pub const fn gateway_symbol(passed: bool, status: CertificationStatus) -> &'static str {
207 if matches!(status, CertificationStatus::Pending) {
208 "-"
209 } else if passed {
210 "\u{2713}" } else {
212 "\u{2717}" }
214 }
215
216 #[must_use]
218 pub fn param_count(&self) -> f64 {
219 self.parameters
220 .trim_end_matches('B')
221 .parse::<f64>()
222 .unwrap_or(0.0)
223 }
224}
225
226const REQUIRED_CSV_COLUMNS: &[&str] = &[
232 "model_id",
233 "family",
234 "parameters",
235 "size_category",
236 "status",
237 "mqs_score",
238 "grade",
239 "certified_tier",
240 "last_certified",
241 "g1",
242 "g2",
243 "g3",
244 "g4",
245];
246
247#[allow(clippy::similar_names)]
254pub fn parse_csv(content: &str) -> Result<Vec<ModelCertification>> {
255 let mut models = Vec::new();
256 let mut lines = content.lines().enumerate();
257
258 let Some((_, header)) = lines.next() else {
260 return Ok(models);
261 };
262
263 let header_fields = csv_split(header);
264 let col: std::collections::HashMap<&str, usize> = header_fields
265 .iter()
266 .enumerate()
267 .map(|(i, name)| (name.as_str(), i))
268 .collect();
269
270 for &name in REQUIRED_CSV_COLUMNS {
272 if !col.contains_key(name) {
273 return Err(CertifyError::CsvParse {
274 line: 1,
275 message: format!("missing required column '{name}'"),
276 });
277 }
278 }
279
280 for (line_num, line) in lines {
281 if line.trim().is_empty() {
282 continue;
283 }
284
285 let fields = csv_split(line);
286 let get = |name: &str| -> &str {
287 col.get(name)
288 .and_then(|&i| fields.get(i).map(String::as_str))
289 .unwrap_or("")
290 };
291 let get_opt = |name: &str| -> Option<String> {
292 col.get(name)
293 .and_then(|&i| fields.get(i))
294 .filter(|s| !s.is_empty())
295 .cloned()
296 };
297
298 let last_certified = DateTime::parse_from_rfc3339(get("last_certified"))
299 .ok()
300 .map(|dt| dt.with_timezone(&Utc));
301
302 models.push(ModelCertification {
303 model_id: get("model_id").to_owned(),
304 family: get("family").to_owned(),
305 parameters: get("parameters").to_owned(),
306 size_category: SizeCategory::parse(get("size_category")).ok_or_else(|| {
307 CertifyError::CsvParse {
308 line: line_num + 1,
309 message: format!("invalid size_category: '{}'", get("size_category")),
310 }
311 })?,
312 status: CertificationStatus::parse(get("status")).ok_or_else(|| {
313 CertifyError::CsvParse {
314 line: line_num + 1,
315 message: format!("invalid status: '{}'", get("status")),
316 }
317 })?,
318 mqs_score: get("mqs_score")
319 .parse()
320 .map_err(|_| CertifyError::CsvParse {
321 line: line_num + 1,
322 message: format!("invalid mqs_score: '{}'", get("mqs_score")),
323 })?,
324 grade: get("grade").to_owned(),
325 certified_tier: get("certified_tier").to_owned(),
326 last_certified,
327 g1: get("g1").to_lowercase() == "true",
328 g2: get("g2").to_lowercase() == "true",
329 g3: get("g3").to_lowercase() == "true",
330 g4: get("g4").to_lowercase() == "true",
331 tps_gguf_cpu: get_opt("tps_gguf_cpu").and_then(|s| s.parse().ok()),
332 tps_gguf_gpu: get_opt("tps_gguf_gpu").and_then(|s| s.parse().ok()),
333 tps_apr_cpu: get_opt("tps_apr_cpu").and_then(|s| s.parse().ok()),
334 tps_apr_gpu: get_opt("tps_apr_gpu").and_then(|s| s.parse().ok()),
335 tps_st_cpu: get_opt("tps_st_cpu").and_then(|s| s.parse().ok()),
336 tps_st_gpu: get_opt("tps_st_gpu").and_then(|s| s.parse().ok()),
337 provenance_verified: get_opt("provenance_verified")
338 .is_some_and(|s| s.to_lowercase() == "true"),
339 kernel_proof_ref: get_opt("kernel_proof_ref"),
340 });
341 }
342
343 Ok(models)
344}
345
346#[must_use]
348pub fn generate_summary(models: &[ModelCertification], timestamp: &str) -> String {
349 let total = models.len();
350 let certified = models
351 .iter()
352 .filter(|m| matches!(m.status, CertificationStatus::Certified))
353 .count();
354 let provisional = models
355 .iter()
356 .filter(|m| matches!(m.status, CertificationStatus::Provisional))
357 .count();
358 let blocked = models
359 .iter()
360 .filter(|m| matches!(m.status, CertificationStatus::Blocked))
361 .count();
362 let pending = models
363 .iter()
364 .filter(|m| matches!(m.status, CertificationStatus::Pending))
365 .count();
366
367 format!(
368 r"**Certification Summary** (updated: {timestamp})
369
370| Status | Count |
371|--------|-------|
372| Certified | {certified}/{total} |
373| Provisional | {provisional}/{total} |
374| Blocked | {blocked}/{total} |
375| Pending | {pending}/{total} |
376
377**Priority Family:** Qwen Coder (see [Certified Testing Spec](docs/specifications/certified-testing.md))"
378 )
379}
380
381#[must_use]
383pub fn generate_table(models: &[ModelCertification]) -> String {
384 let mut lines = Vec::new();
385
386 lines.push(
388 "| Model | Family | Size | Status | MQS | Grade | G1-4 | Prov | GGUF CPU | GGUF GPU | APR CPU | APR GPU | ST CPU | ST GPU |"
389 .to_string(),
390 );
391 lines.push(
392 "|-------|--------|------|--------|-----|-------|------|------|----------|----------|---------|---------|--------|--------|"
393 .to_string(),
394 );
395
396 let mut sorted: Vec<_> = models.iter().collect();
398 sorted.sort_by(|a, b| {
399 a.family.cmp(&b.family).then_with(|| {
400 a.param_count()
401 .partial_cmp(&b.param_count())
402 .unwrap_or(std::cmp::Ordering::Equal)
403 })
404 });
405
406 for m in sorted {
407 let gateways = if matches!(m.status, CertificationStatus::Pending) {
409 "-".to_string()
410 } else if m.g1 && m.g2 && m.g3 && m.g4 {
411 "\u{2713}".to_string() } else {
413 "\u{2717}".to_string() };
415
416 let prov = if matches!(m.status, CertificationStatus::Pending) {
418 "-"
419 } else if m.provenance_verified {
420 "\u{2713}" } else {
422 "\u{2717}" };
424
425 let fmt = |v: Option<f64>| v.map_or_else(|| "-".to_string(), |x| format!("{x:.1}"));
427
428 lines.push(format!(
429 "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |",
430 m.markdown_link(),
431 m.family,
432 m.parameters,
433 m.status.badge(),
434 m.mqs_score,
435 m.grade,
436 gateways,
437 prov,
438 fmt(m.tps_gguf_cpu),
439 fmt(m.tps_gguf_gpu),
440 fmt(m.tps_apr_cpu),
441 fmt(m.tps_apr_gpu),
442 fmt(m.tps_st_cpu),
443 fmt(m.tps_st_gpu),
444 ));
445 }
446
447 lines.join("\n")
448}
449
450pub const START_MARKER: &str = "<!-- CERTIFICATION_TABLE_START -->";
452pub const END_MARKER: &str = "<!-- CERTIFICATION_TABLE_END -->";
454
455#[must_use]
459pub fn write_csv(models: &[ModelCertification]) -> String {
460 let mut lines = Vec::new();
461
462 lines.push(
464 "model_id,family,parameters,size_category,status,mqs_score,grade,certified_tier,last_certified,g1,g2,g3,g4,tps_gguf_cpu,tps_gguf_gpu,tps_apr_cpu,tps_apr_gpu,tps_st_cpu,tps_st_gpu,provenance_verified,kernel_proof_ref"
465 .to_string(),
466 );
467
468 for m in models {
469 let size_cat = match m.size_category {
470 SizeCategory::Tiny => "tiny",
471 SizeCategory::Small => "small",
472 SizeCategory::Medium => "medium",
473 SizeCategory::Large => "large",
474 SizeCategory::XLarge => "xlarge",
475 };
476 let last_cert = m
477 .last_certified
478 .map_or_else(String::new, |dt| dt.to_rfc3339());
479
480 let fmt = |v: Option<f64>| v.map_or(String::new(), |x| format!("{x:.1}"));
482
483 lines.push(format!(
484 "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}",
485 csv_quote(&m.model_id),
486 csv_quote(&m.family),
487 csv_quote(&m.parameters),
488 size_cat,
489 m.status,
490 m.mqs_score,
491 csv_quote(&m.grade),
492 csv_quote(&m.certified_tier),
493 last_cert,
494 m.g1,
495 m.g2,
496 m.g3,
497 m.g4,
498 fmt(m.tps_gguf_cpu),
499 fmt(m.tps_gguf_gpu),
500 fmt(m.tps_apr_cpu),
501 fmt(m.tps_apr_gpu),
502 fmt(m.tps_st_cpu),
503 fmt(m.tps_st_gpu),
504 m.provenance_verified,
505 csv_quote(m.kernel_proof_ref.as_deref().unwrap_or("")),
506 ));
507 }
508
509 lines.join("\n") + "\n"
510}
511
512fn csv_quote(field: &str) -> String {
516 if field.contains(',') || field.contains('"') || field.contains('\n') || field.contains('\r') {
517 format!("\"{}\"", field.replace('"', "\"\""))
518 } else {
519 field.to_string()
520 }
521}
522
523fn csv_split(line: &str) -> Vec<String> {
526 let mut fields = Vec::new();
527 let mut current = String::new();
528 let mut in_quotes = false;
529 let mut chars = line.chars().peekable();
530
531 while let Some(ch) = chars.next() {
532 match ch {
533 '"' if in_quotes => {
534 if chars.peek() == Some(&'"') {
535 current.push('"');
537 chars.next();
538 } else {
539 in_quotes = false;
540 }
541 }
542 '"' if !in_quotes && current.is_empty() => {
543 in_quotes = true;
544 }
545 ',' if !in_quotes => {
546 fields.push(std::mem::take(&mut current));
547 }
548 _ => {
549 current.push(ch);
550 }
551 }
552 }
553 fields.push(current);
554 fields
555}
556
557include!("lib_scoring.rs");
559
560#[cfg(test)]
561#[path = "lib_tests.rs"]
562mod tests;