fttsq_census/
fttsq_census.rs

1//! Byte census of a `.fttsq` artifact: exact composition by section, access class, and dtype.
2//!
3//! The consumer is the artifact-v2 quantization program (`frankentts-fs6z`): every "quantize X
4//! next" decision is ordered by measured share of the file, and this is the measurement. NDJSON
5//! on stdout, one row per aggregate, `kind` first so a stream consumer can route rows; nothing
6//! human-decorated (AGENTS.md agent-ergonomics conventions).
7//!
8//! ```text
9//! cargo run -p ftts-artifacts --example fttsq_census -- ~/.cache/franken_tts/model/qwen3-tts-12hz-0.6b-base.fttsq
10//! ```
11//!
12//! Opening via [`MappedFttsq::open`] means every reported byte is digest-verified before it is
13//! counted — a census of a corrupt artifact is refused, not reported.
14
15use ftts_artifacts::fttsq::MappedFttsq;
16use std::collections::BTreeMap;
17
18fn row(kind: &str, fields: &[(&str, String)]) {
19    let mut line = String::from("{\"kind\":\"");
20    line.push_str(kind);
21    line.push('"');
22    for (key, value) in fields {
23        line.push_str(",\"");
24        line.push_str(key);
25        line.push_str("\":");
26        line.push_str(value);
27    }
28    line.push('}');
29    println!("{line}");
30}
31
32fn quoted(text: &str) -> String {
33    // Section/tensor names are ASCII identifiers by construction; no escaping cases exist.
34    format!("\"{text}\"")
35}
36
37fn share(bytes: u64, total: u64) -> String {
38    format!("{:.4}", bytes as f64 / total as f64)
39}
40
41fn main() {
42    let path = std::env::args().nth(1).unwrap_or_else(|| {
43        eprintln!("usage: fttsq_census <artifact.fttsq>");
44        std::process::exit(2);
45    });
46    let mapped = match MappedFttsq::open(&path) {
47        Ok(mapped) => mapped,
48        Err(error) => {
49            eprintln!("refusing census: {error}");
50            std::process::exit(1);
51        }
52    };
53    let reader = mapped.reader();
54    let file_bytes = mapped.len() as u64;
55
56    let mut class_bytes: BTreeMap<&str, u64> = BTreeMap::new();
57    let mut sections_total = 0_u64;
58    for section in reader.sections() {
59        sections_total += section.length;
60        *class_bytes
61            .entry(section.access_class.as_str())
62            .or_default() += section.length;
63        row(
64            "section",
65            &[
66                ("name", quoted(&section.name)),
67                ("access_class", quoted(section.access_class.as_str())),
68                ("bytes", section.length.to_string()),
69                ("share_of_file", share(section.length, file_bytes)),
70            ],
71        );
72    }
73
74    // Dtype split inside each section: separates payload from quantization-scale overhead and
75    // shows how much of a "quantized" section is still wide.
76    let mut dtype_bytes: BTreeMap<(String, &str), u64> = BTreeMap::new();
77    let mut scale_bytes: BTreeMap<String, u64> = BTreeMap::new();
78    let scale_names: std::collections::BTreeSet<&str> = reader
79        .tensors()
80        .iter()
81        .filter_map(|tensor| tensor.scales.as_deref())
82        .collect();
83    for tensor in reader.tensors() {
84        *dtype_bytes
85            .entry((tensor.section.clone(), tensor.dtype.as_str()))
86            .or_default() += tensor.length;
87        if scale_names.contains(tensor.name.as_str()) {
88            *scale_bytes.entry(tensor.section.clone()).or_default() += tensor.length;
89        }
90    }
91    for ((section, dtype), bytes) in &dtype_bytes {
92        row(
93            "section_dtype",
94            &[
95                ("section", quoted(section)),
96                ("dtype", quoted(dtype)),
97                ("bytes", bytes.to_string()),
98                ("share_of_file", share(*bytes, file_bytes)),
99            ],
100        );
101    }
102    for (section, bytes) in &scale_bytes {
103        row(
104            "scale_overhead",
105            &[
106                ("section", quoted(section)),
107                ("bytes", bytes.to_string()),
108                ("share_of_file", share(*bytes, file_bytes)),
109            ],
110        );
111    }
112
113    for (class, bytes) in &class_bytes {
114        row(
115            "access_class",
116            &[
117                ("access_class", quoted(class)),
118                ("bytes", bytes.to_string()),
119                ("share_of_file", share(*bytes, file_bytes)),
120            ],
121        );
122    }
123    row(
124        "total",
125        &[
126            ("file_bytes", file_bytes.to_string()),
127            ("section_bytes", sections_total.to_string()),
128            (
129                "header_directory_bytes",
130                (file_bytes - sections_total).to_string(),
131            ),
132            ("model_family", quoted(reader.model_family())),
133        ],
134    );
135}
fttsq_census/fttsq_census.rs

fttsq_census/
fttsq_census.rs
