Skip to main content

aprender_qa_certify/
lib.rs

1//! Model certification tools and README synchronization.
2//!
3//! This crate provides utilities for:
4//! - Parsing model certification CSV data
5//! - Generating markdown tables for README
6//! - Synchronizing certification status with documentation
7
8#![forbid(unsafe_code)]
9#![cfg_attr(
10    test,
11    allow(
12        clippy::unwrap_used,
13        clippy::expect_used,
14        clippy::panic,
15        clippy::doc_markdown
16    )
17)]
18
19use chrono::{DateTime, Utc};
20use std::fmt;
21use thiserror::Error;
22
23/// Errors that can occur during certification operations.
24#[derive(Error, Debug)]
25pub enum CertifyError {
26    /// CSV parsing error.
27    #[error("CSV parse error at line {line}: {message}")]
28    CsvParse {
29        /// Line number where error occurred.
30        line: usize,
31        /// Error message.
32        message: String,
33    },
34
35    /// README marker not found.
36    #[error("README marker not found: {0}")]
37    MarkerNotFound(String),
38
39    /// IO error.
40    #[error("IO error: {0}")]
41    Io(#[from] std::io::Error),
42}
43
44/// Result type for certification operations.
45pub type Result<T> = std::result::Result<T, CertifyError>;
46
47/// Certification status for a model.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
49pub enum CertificationStatus {
50    /// Model passed all tests with MQS >= 850.
51    Certified,
52    /// Model passed with MQS >= 700 but < 850.
53    Provisional,
54    /// Model failed tests or MQS < 700.
55    Blocked,
56    /// Model not yet tested.
57    #[default]
58    Pending,
59}
60
61impl CertificationStatus {
62    /// Parse status from string representation.
63    /// Returns `None` for unrecognized values (Jidoka: fail loud on bad data).
64    #[must_use]
65    pub fn parse(s: &str) -> Option<Self> {
66        match s.to_uppercase().as_str() {
67            "CERTIFIED" => Some(Self::Certified),
68            "PROVISIONAL" => Some(Self::Provisional),
69            "BLOCKED" => Some(Self::Blocked),
70            "PENDING" => Some(Self::Pending),
71            _ => None,
72        }
73    }
74
75    /// Get badge markdown for this status.
76    #[must_use]
77    pub const fn badge(&self) -> &'static str {
78        match self {
79            Self::Certified => "![certified](https://img.shields.io/badge/CERTIFIED-brightgreen)",
80            Self::Provisional => "![provisional](https://img.shields.io/badge/PROVISIONAL-yellow)",
81            Self::Blocked => "![blocked](https://img.shields.io/badge/BLOCKED-red)",
82            Self::Pending => "![pending](https://img.shields.io/badge/PENDING-lightgray)",
83        }
84    }
85}
86
87impl fmt::Display for CertificationStatus {
88    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89        match self {
90            Self::Certified => write!(f, "CERTIFIED"),
91            Self::Provisional => write!(f, "PROVISIONAL"),
92            Self::Blocked => write!(f, "BLOCKED"),
93            Self::Pending => write!(f, "PENDING"),
94        }
95    }
96}
97
98/// Size category for models.
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
100pub enum SizeCategory {
101    /// < 1B parameters.
102    Tiny,
103    /// 1B - 5B parameters.
104    #[default]
105    Small,
106    /// 5B - 10B parameters.
107    Medium,
108    /// 10B - 30B parameters.
109    Large,
110    /// > 30B parameters.
111    XLarge,
112}
113
114impl SizeCategory {
115    /// Parse size category from string representation.
116    /// Returns `None` for unrecognized values (Jidoka: fail loud on bad data).
117    #[must_use]
118    pub fn parse(s: &str) -> Option<Self> {
119        match s.to_lowercase().as_str() {
120            "tiny" => Some(Self::Tiny),
121            "small" => Some(Self::Small),
122            "medium" => Some(Self::Medium),
123            "large" => Some(Self::Large),
124            "xlarge" => Some(Self::XLarge),
125            _ => None,
126        }
127    }
128}
129
130/// Model certification record.
131///
132/// Contains certification data for a single model including gateway status
133/// and throughput measurements per format.
134/// The four gateway bools (g1-g4) are required for the certification protocol.
135#[allow(clippy::struct_excessive_bools)]
136#[derive(Debug, Clone)]
137pub struct ModelCertification {
138    /// `HuggingFace` model ID.
139    pub model_id: String,
140    /// Model family (e.g., qwen-coder, llama).
141    pub family: String,
142    /// Parameter count (e.g., "1.5B").
143    pub parameters: String,
144    /// Size category.
145    pub size_category: SizeCategory,
146    /// Certification status.
147    pub status: CertificationStatus,
148    /// Model Qualification Score (0-1000).
149    pub mqs_score: u32,
150    /// Letter grade.
151    pub grade: String,
152    /// Highest passing tier.
153    pub certified_tier: String,
154    /// Last certification timestamp.
155    pub last_certified: Option<DateTime<Utc>>,
156    /// Gateway 1 (load) status.
157    pub g1: bool,
158    /// Gateway 2 (inference) status.
159    pub g2: bool,
160    /// Gateway 3 (stability) status.
161    pub g3: bool,
162    /// Gateway 4 (quality) status.
163    pub g4: bool,
164    /// Throughput in tokens/sec for GGUF format (CPU).
165    pub tps_gguf_cpu: Option<f64>,
166    /// Throughput in tokens/sec for GGUF format (GPU).
167    pub tps_gguf_gpu: Option<f64>,
168    /// Throughput in tokens/sec for APR format (CPU).
169    pub tps_apr_cpu: Option<f64>,
170    /// Throughput in tokens/sec for APR format (GPU).
171    pub tps_apr_gpu: Option<f64>,
172    /// Throughput in tokens/sec for `SafeTensors` format (CPU).
173    pub tps_st_cpu: Option<f64>,
174    /// Throughput in tokens/sec for `SafeTensors` format (GPU).
175    pub tps_st_gpu: Option<f64>,
176    /// Provenance verified (PMAT-PROV-001).
177    pub provenance_verified: bool,
178    /// Kernel proof reference model (for dim-smoke tier).
179    pub kernel_proof_ref: Option<String>,
180}
181
182impl ModelCertification {
183    /// Get the short model name (without org prefix).
184    #[must_use]
185    pub fn short_name(&self) -> &str {
186        self.model_id
187            .split('/')
188            .next_back()
189            .unwrap_or(&self.model_id)
190    }
191
192    /// Get `HuggingFace` URL for this model.
193    #[must_use]
194    pub fn hf_url(&self) -> String {
195        format!("https://huggingface.co/{}", self.model_id)
196    }
197
198    /// Get markdown link for this model.
199    #[must_use]
200    pub fn markdown_link(&self) -> String {
201        format!("[{}]({})", self.short_name(), self.hf_url())
202    }
203
204    /// Get gateway symbol for display.
205    #[must_use]
206    pub const fn gateway_symbol(passed: bool, status: CertificationStatus) -> &'static str {
207        if matches!(status, CertificationStatus::Pending) {
208            "-"
209        } else if passed {
210            "\u{2713}" // checkmark
211        } else {
212            "\u{2717}" // x mark
213        }
214    }
215
216    /// Parse numeric parameter count for sorting.
217    #[must_use]
218    pub fn param_count(&self) -> f64 {
219        self.parameters
220            .trim_end_matches('B')
221            .parse::<f64>()
222            .unwrap_or(0.0)
223    }
224}
225
226/// Parse CSV content into model certifications.
227///
228/// # Errors
229///
230/// Required CSV columns — must all be present in the header.
231const REQUIRED_CSV_COLUMNS: &[&str] = &[
232    "model_id",
233    "family",
234    "parameters",
235    "size_category",
236    "status",
237    "mqs_score",
238    "grade",
239    "certified_tier",
240    "last_certified",
241    "g1",
242    "g2",
243    "g3",
244    "g4",
245];
246
247/// Parse a CSV certification table into a list of `ModelCertification` records.
248///
249/// # Errors
250///
251/// Returns `CertifyError::CsvParse` if the CSV header is missing required columns
252/// or if any data row contains malformed field values.
253#[allow(clippy::similar_names)]
254pub fn parse_csv(content: &str) -> Result<Vec<ModelCertification>> {
255    let mut models = Vec::new();
256    let mut lines = content.lines().enumerate();
257
258    // Parse header to build name→index map (prevents fragility from column reordering)
259    let Some((_, header)) = lines.next() else {
260        return Ok(models);
261    };
262
263    let header_fields = csv_split(header);
264    let col: std::collections::HashMap<&str, usize> = header_fields
265        .iter()
266        .enumerate()
267        .map(|(i, name)| (name.as_str(), i))
268        .collect();
269
270    // Fail fast if any required columns are missing
271    for &name in REQUIRED_CSV_COLUMNS {
272        if !col.contains_key(name) {
273            return Err(CertifyError::CsvParse {
274                line: 1,
275                message: format!("missing required column '{name}'"),
276            });
277        }
278    }
279
280    for (line_num, line) in lines {
281        if line.trim().is_empty() {
282            continue;
283        }
284
285        let fields = csv_split(line);
286        let get = |name: &str| -> &str {
287            col.get(name)
288                .and_then(|&i| fields.get(i).map(String::as_str))
289                .unwrap_or("")
290        };
291        let get_opt = |name: &str| -> Option<String> {
292            col.get(name)
293                .and_then(|&i| fields.get(i))
294                .filter(|s| !s.is_empty())
295                .cloned()
296        };
297
298        let last_certified = DateTime::parse_from_rfc3339(get("last_certified"))
299            .ok()
300            .map(|dt| dt.with_timezone(&Utc));
301
302        models.push(ModelCertification {
303            model_id: get("model_id").to_owned(),
304            family: get("family").to_owned(),
305            parameters: get("parameters").to_owned(),
306            size_category: SizeCategory::parse(get("size_category")).ok_or_else(|| {
307                CertifyError::CsvParse {
308                    line: line_num + 1,
309                    message: format!("invalid size_category: '{}'", get("size_category")),
310                }
311            })?,
312            status: CertificationStatus::parse(get("status")).ok_or_else(|| {
313                CertifyError::CsvParse {
314                    line: line_num + 1,
315                    message: format!("invalid status: '{}'", get("status")),
316                }
317            })?,
318            mqs_score: get("mqs_score")
319                .parse()
320                .map_err(|_| CertifyError::CsvParse {
321                    line: line_num + 1,
322                    message: format!("invalid mqs_score: '{}'", get("mqs_score")),
323                })?,
324            grade: get("grade").to_owned(),
325            certified_tier: get("certified_tier").to_owned(),
326            last_certified,
327            g1: get("g1").to_lowercase() == "true",
328            g2: get("g2").to_lowercase() == "true",
329            g3: get("g3").to_lowercase() == "true",
330            g4: get("g4").to_lowercase() == "true",
331            tps_gguf_cpu: get_opt("tps_gguf_cpu").and_then(|s| s.parse().ok()),
332            tps_gguf_gpu: get_opt("tps_gguf_gpu").and_then(|s| s.parse().ok()),
333            tps_apr_cpu: get_opt("tps_apr_cpu").and_then(|s| s.parse().ok()),
334            tps_apr_gpu: get_opt("tps_apr_gpu").and_then(|s| s.parse().ok()),
335            tps_st_cpu: get_opt("tps_st_cpu").and_then(|s| s.parse().ok()),
336            tps_st_gpu: get_opt("tps_st_gpu").and_then(|s| s.parse().ok()),
337            provenance_verified: get_opt("provenance_verified")
338                .is_some_and(|s| s.to_lowercase() == "true"),
339            kernel_proof_ref: get_opt("kernel_proof_ref"),
340        });
341    }
342
343    Ok(models)
344}
345
346/// Generate certification summary statistics.
347#[must_use]
348pub fn generate_summary(models: &[ModelCertification], timestamp: &str) -> String {
349    let total = models.len();
350    let certified = models
351        .iter()
352        .filter(|m| matches!(m.status, CertificationStatus::Certified))
353        .count();
354    let provisional = models
355        .iter()
356        .filter(|m| matches!(m.status, CertificationStatus::Provisional))
357        .count();
358    let blocked = models
359        .iter()
360        .filter(|m| matches!(m.status, CertificationStatus::Blocked))
361        .count();
362    let pending = models
363        .iter()
364        .filter(|m| matches!(m.status, CertificationStatus::Pending))
365        .count();
366
367    format!(
368        r"**Certification Summary** (updated: {timestamp})
369
370| Status | Count |
371|--------|-------|
372| Certified | {certified}/{total} |
373| Provisional | {provisional}/{total} |
374| Blocked | {blocked}/{total} |
375| Pending | {pending}/{total} |
376
377**Priority Family:** Qwen Coder (see [Certified Testing Spec](docs/specifications/certified-testing.md))"
378    )
379}
380
381/// Generate markdown table from model certifications.
382#[must_use]
383pub fn generate_table(models: &[ModelCertification]) -> String {
384    let mut lines = Vec::new();
385
386    // Header with tok/s columns (format × backend = 6 columns) + provenance
387    lines.push(
388        "| Model | Family | Size | Status | MQS | Grade | G1-4 | Prov | GGUF CPU | GGUF GPU | APR CPU | APR GPU | ST CPU | ST GPU |"
389            .to_string(),
390    );
391    lines.push(
392        "|-------|--------|------|--------|-----|-------|------|------|----------|----------|---------|---------|--------|--------|"
393            .to_string(),
394    );
395
396    // Sort by family, then by parameter count
397    let mut sorted: Vec<_> = models.iter().collect();
398    sorted.sort_by(|a, b| {
399        a.family.cmp(&b.family).then_with(|| {
400            a.param_count()
401                .partial_cmp(&b.param_count())
402                .unwrap_or(std::cmp::Ordering::Equal)
403        })
404    });
405
406    for m in sorted {
407        // Combine gateways into single column (all must pass)
408        let gateways = if matches!(m.status, CertificationStatus::Pending) {
409            "-".to_string()
410        } else if m.g1 && m.g2 && m.g3 && m.g4 {
411            "\u{2713}".to_string() // checkmark
412        } else {
413            "\u{2717}".to_string() // x mark
414        };
415
416        // Provenance status
417        let prov = if matches!(m.status, CertificationStatus::Pending) {
418            "-"
419        } else if m.provenance_verified {
420            "\u{2713}" // checkmark
421        } else {
422            "\u{2717}" // x mark
423        };
424
425        // Format tok/s values (6 columns)
426        let fmt = |v: Option<f64>| v.map_or_else(|| "-".to_string(), |x| format!("{x:.1}"));
427
428        lines.push(format!(
429            "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |",
430            m.markdown_link(),
431            m.family,
432            m.parameters,
433            m.status.badge(),
434            m.mqs_score,
435            m.grade,
436            gateways,
437            prov,
438            fmt(m.tps_gguf_cpu),
439            fmt(m.tps_gguf_gpu),
440            fmt(m.tps_apr_cpu),
441            fmt(m.tps_apr_gpu),
442            fmt(m.tps_st_cpu),
443            fmt(m.tps_st_gpu),
444        ));
445    }
446
447    lines.join("\n")
448}
449
450/// Markers for README table replacement.
451pub const START_MARKER: &str = "<!-- CERTIFICATION_TABLE_START -->";
452/// End marker for README table.
453pub const END_MARKER: &str = "<!-- CERTIFICATION_TABLE_END -->";
454
455/// Write certification records to CSV format.
456///
457/// Generates a CSV string with headers that can be written to models.csv.
458#[must_use]
459pub fn write_csv(models: &[ModelCertification]) -> String {
460    let mut lines = Vec::new();
461
462    // Header with 6 tps columns (format × backend) + provenance + kernel_proof_ref
463    lines.push(
464        "model_id,family,parameters,size_category,status,mqs_score,grade,certified_tier,last_certified,g1,g2,g3,g4,tps_gguf_cpu,tps_gguf_gpu,tps_apr_cpu,tps_apr_gpu,tps_st_cpu,tps_st_gpu,provenance_verified,kernel_proof_ref"
465            .to_string(),
466    );
467
468    for m in models {
469        let size_cat = match m.size_category {
470            SizeCategory::Tiny => "tiny",
471            SizeCategory::Small => "small",
472            SizeCategory::Medium => "medium",
473            SizeCategory::Large => "large",
474            SizeCategory::XLarge => "xlarge",
475        };
476        let last_cert = m
477            .last_certified
478            .map_or_else(String::new, |dt| dt.to_rfc3339());
479
480        // Format tps values (empty string for None)
481        let fmt = |v: Option<f64>| v.map_or(String::new(), |x| format!("{x:.1}"));
482
483        lines.push(format!(
484            "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}",
485            csv_quote(&m.model_id),
486            csv_quote(&m.family),
487            csv_quote(&m.parameters),
488            size_cat,
489            m.status,
490            m.mqs_score,
491            csv_quote(&m.grade),
492            csv_quote(&m.certified_tier),
493            last_cert,
494            m.g1,
495            m.g2,
496            m.g3,
497            m.g4,
498            fmt(m.tps_gguf_cpu),
499            fmt(m.tps_gguf_gpu),
500            fmt(m.tps_apr_cpu),
501            fmt(m.tps_apr_gpu),
502            fmt(m.tps_st_cpu),
503            fmt(m.tps_st_gpu),
504            m.provenance_verified,
505            csv_quote(m.kernel_proof_ref.as_deref().unwrap_or("")),
506        ));
507    }
508
509    lines.join("\n") + "\n"
510}
511
512/// RFC 4180 CSV quoting: wrap in double-quotes if the field contains
513/// a comma, double-quote, or newline. Internal double-quotes are escaped
514/// by doubling them.
515fn csv_quote(field: &str) -> String {
516    if field.contains(',') || field.contains('"') || field.contains('\n') || field.contains('\r') {
517        format!("\"{}\"", field.replace('"', "\"\""))
518    } else {
519        field.to_string()
520    }
521}
522
523/// RFC 4180 CSV field splitting: handles double-quoted fields containing
524/// commas and escaped double-quotes (doubled `""`).
525fn csv_split(line: &str) -> Vec<String> {
526    let mut fields = Vec::new();
527    let mut current = String::new();
528    let mut in_quotes = false;
529    let mut chars = line.chars().peekable();
530
531    while let Some(ch) = chars.next() {
532        match ch {
533            '"' if in_quotes => {
534                if chars.peek() == Some(&'"') {
535                    // Escaped double-quote
536                    current.push('"');
537                    chars.next();
538                } else {
539                    in_quotes = false;
540                }
541            }
542            '"' if !in_quotes && current.is_empty() => {
543                in_quotes = true;
544            }
545            ',' if !in_quotes => {
546                fields.push(std::mem::take(&mut current));
547            }
548            _ => {
549                current.push(ch);
550            }
551        }
552    }
553    fields.push(current);
554    fields
555}
556
557// Certification tier scoring logic
558include!("lib_scoring.rs");
559
560#[cfg(test)]
561#[path = "lib_tests.rs"]
562mod tests;