Skip to main content

text_mirror/
manifest.rs

1//! Serde types for the manifest schema `text-mirror/manifest@1`, the
2//! append-only JSONL writer, and the shard reader.
3//!
4//! The manifest is the public interface of this crate. Each division
5//! writes one shard, `manifest/<division>.jsonl`. One line is one
6//! [`Record`]. Shards are append-only, and the latest record for a
7//! source path is its terminal outcome.
8//!
9//! Every record self-describes its schema in its first field, so a
10//! shard separated from its bundle stays verifiable. A consumer must
11//! reject any record whose `schema` value it does not recognize.
12//! [`ManifestSchema`] enforces both rules at the serde layer, and
13//! records reject unknown fields, so an additive change is a breaking
14//! change that bumps the schema version.
15
16use std::fs::{self, File, OpenOptions};
17use std::io::Write;
18use std::path::{Path, PathBuf};
19
20use serde::de::Error as _;
21use serde::{Deserialize, Deserializer, Serialize, Serializer};
22
23use crate::{Error, Result};
24
25/// The schema identifier carried by every manifest@1 record.
26pub const MANIFEST_SCHEMA: &str = "text-mirror/manifest@1";
27
28/// The `schema` field of a [`Record`].
29///
30/// Serializes to the literal `text-mirror/manifest@1`. Deserialization
31/// rejects every other value, so a consumer of these types fails
32/// closed on a schema version it does not recognize.
33#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
34pub struct ManifestSchema;
35
36impl Serialize for ManifestSchema {
37    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
38        serializer.serialize_str(MANIFEST_SCHEMA)
39    }
40}
41
42impl<'de> Deserialize<'de> for ManifestSchema {
43    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
44        let value = String::deserialize(deserializer)?;
45        if value == MANIFEST_SCHEMA {
46            Ok(ManifestSchema)
47        } else {
48            Err(D::Error::custom(format!(
49                "unrecognized schema {value:?}, this consumer accepts only {MANIFEST_SCHEMA:?}"
50            )))
51        }
52    }
53}
54
55/// The outcome recorded for one source file.
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
57#[serde(rename_all = "snake_case")]
58pub enum Status {
59    /// A converter produced a text artifact.
60    Converted,
61    /// A converter ran and failed. `error` names the reason.
62    Failed,
63    /// No converter claims the detected format.
64    Unsupported,
65    /// The checkpoint key matched a prior record, so no work was done.
66    SkippedUnchanged,
67    /// The source hash matched an already converted source. The text
68    /// artifact is a copy, and `dedup_of` names the canonical source.
69    Dedup,
70}
71
72/// How a text artifact was derived from its source.
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
74#[serde(rename_all = "snake_case")]
75pub enum ArtifactKind {
76    /// Text extracted or passed through from a text-bearing source.
77    Text,
78    /// A transcript derived from audio or video.
79    Transcript,
80    /// Text recognized from images.
81    Ocr,
82    /// A combination of the other kinds.
83    Mixed,
84}
85
86/// Media provenance for artifacts derived from audio or video.
87#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
88#[serde(deny_unknown_fields)]
89pub struct Media {
90    /// Source duration in seconds.
91    #[serde(skip_serializing_if = "Option::is_none", default)]
92    pub duration_seconds: Option<f64>,
93    /// Detected language tag.
94    #[serde(skip_serializing_if = "Option::is_none", default)]
95    pub language: Option<String>,
96    /// Transcription model id.
97    #[serde(skip_serializing_if = "Option::is_none", default)]
98    pub model_id: Option<String>,
99    /// Hash of the transcription model.
100    #[serde(skip_serializing_if = "Option::is_none", default)]
101    pub model_hash: Option<String>,
102    /// Hash of the pinned decode options.
103    #[serde(skip_serializing_if = "Option::is_none", default)]
104    pub decode_options_hash: Option<String>,
105    /// Number of distinct speakers.
106    #[serde(skip_serializing_if = "Option::is_none", default)]
107    pub speaker_count: Option<u32>,
108}
109
110/// One manifest@1 record. One line of a JSONL shard.
111///
112/// Field names and their JSON spellings are the frozen contract.
113/// Unknown fields are rejected on read.
114#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
115#[serde(deny_unknown_fields)]
116pub struct Record {
117    /// The literal string `text-mirror/manifest@1`, always first.
118    pub schema: ManifestSchema,
119    /// Source path relative to the division root.
120    pub source_path: String,
121    /// Lowercase hex BLAKE3 hash of the source bytes.
122    pub source_hash: String,
123    /// Source size in bytes.
124    pub source_size: u64,
125    /// Format id the file extension declares, when known.
126    #[serde(skip_serializing_if = "Option::is_none", default)]
127    pub declared_format: Option<String>,
128    /// Resolved format id from detection.
129    pub detected_format: String,
130    /// True when magic bytes and the extension disagree.
131    pub format_mismatch: bool,
132    /// The recorded outcome.
133    pub status: Status,
134    /// Text artifact path relative to the mirror root, beginning with
135    /// the record's division segment: `<division>/<source_path>.txt`.
136    #[serde(skip_serializing_if = "Option::is_none", default)]
137    pub text_path: Option<String>,
138    /// Lowercase hex BLAKE3 hash of the text artifact bytes.
139    #[serde(skip_serializing_if = "Option::is_none", default)]
140    pub text_hash: Option<String>,
141    /// How the artifact was derived.
142    #[serde(skip_serializing_if = "Option::is_none", default)]
143    pub artifact_kind: Option<ArtifactKind>,
144    /// Converter that handled the source.
145    #[serde(skip_serializing_if = "Option::is_none", default)]
146    pub converter_id: Option<String>,
147    /// Version of that converter.
148    #[serde(skip_serializing_if = "Option::is_none", default)]
149    pub converter_version: Option<String>,
150    /// Version of an external tool the converter wrapped, when any.
151    #[serde(skip_serializing_if = "Option::is_none", default)]
152    pub tool_version: Option<String>,
153    /// Version of the rules files the run used.
154    pub rules_version: String,
155    /// Media provenance for transcript artifacts.
156    #[serde(skip_serializing_if = "Option::is_none", default)]
157    pub media: Option<Media>,
158    /// Container source this record was expanded from, when any.
159    #[serde(skip_serializing_if = "Option::is_none", default)]
160    pub parent_source: Option<String>,
161    /// Canonical source path for a `dedup` record.
162    #[serde(skip_serializing_if = "Option::is_none", default)]
163    pub dedup_of: Option<String>,
164    /// Non-fatal notes from conversion.
165    #[serde(skip_serializing_if = "Vec::is_empty", default)]
166    pub warnings: Vec<String>,
167    /// Machine-readable reason for a `failed` record, or the declared
168    /// capability gap on an `unsupported` record. Forbidden on every
169    /// other status.
170    #[serde(skip_serializing_if = "Option::is_none", default)]
171    pub error: Option<String>,
172    /// Wall-clock milliseconds spent on this source.
173    #[serde(skip_serializing_if = "Option::is_none", default)]
174    pub duration_ms: Option<u64>,
175}
176
177fn status_name(status: Status) -> &'static str {
178    match status {
179        Status::Converted => "converted",
180        Status::Failed => "failed",
181        Status::Unsupported => "unsupported",
182        Status::SkippedUnchanged => "skipped_unchanged",
183        Status::Dedup => "dedup",
184    }
185}
186
187impl Record {
188    /// Checks the cross-field invariants of the schema.
189    ///
190    /// A record with a text-bearing status must carry `text_path` and
191    /// `text_hash`. A `failed` or `unsupported` record must not. A
192    /// `dedup` record must name `dedup_of`, and a `failed` record must
193    /// name `error`. An `unsupported` record may carry `error` to name
194    /// a declared capability gap, such as a spreadsheet format with no
195    /// visibility reader yet.
196    pub fn validate(&self) -> std::result::Result<(), String> {
197        let name = status_name(self.status);
198        match self.status {
199            Status::Converted | Status::SkippedUnchanged | Status::Dedup => {
200                if self.text_path.is_none() || self.text_hash.is_none() {
201                    return Err(format!("status {name} requires text_path and text_hash"));
202                }
203                if self.error.is_some() {
204                    return Err(format!("status {name} forbids error"));
205                }
206            }
207            Status::Failed | Status::Unsupported => {
208                if self.text_path.is_some() || self.text_hash.is_some() {
209                    return Err(format!("status {name} forbids text_path and text_hash"));
210                }
211            }
212        }
213        if self.status == Status::Dedup && self.dedup_of.is_none() {
214            return Err("status dedup requires dedup_of".to_string());
215        }
216        if self.status == Status::Failed && self.error.is_none() {
217            return Err("status failed requires error".to_string());
218        }
219        Ok(())
220    }
221}
222
223/// Appends records to a JSONL shard. One record per line, LF only.
224pub struct ManifestWriter {
225    path: PathBuf,
226    file: File,
227}
228
229impl ManifestWriter {
230    /// Opens a shard for append, creating parent directories.
231    ///
232    /// A record never starts on a line that lacks a trailing LF. When
233    /// the existing shard ends mid-line, a complete final record gets
234    /// its missing LF, and a torn partial tail from a killed run is
235    /// dropped so the next run re-records that source.
236    pub fn open(path: &Path) -> Result<Self> {
237        if let Some(dir) = path.parent() {
238            fs::create_dir_all(dir).map_err(|e| Error::io("create_dir", dir, e))?;
239        }
240        repair_tail(path)?;
241        let file = OpenOptions::new()
242            .create(true)
243            .append(true)
244            .open(path)
245            .map_err(|e| Error::io("open", path, e))?;
246        Ok(ManifestWriter {
247            path: path.to_path_buf(),
248            file,
249        })
250    }
251
252    /// Serializes one record and appends it as one line.
253    pub fn append(&mut self, record: &Record) -> Result<()> {
254        let mut line = serde_json::to_string(record).map_err(|e| Error::Encode {
255            path: self.path.clone(),
256            message: e.to_string(),
257        })?;
258        line.push('\n');
259        self.file
260            .write_all(line.as_bytes())
261            .map_err(|e| Error::io("append", &self.path, e))?;
262        self.file
263            .flush()
264            .map_err(|e| Error::io("flush", &self.path, e))?;
265        Ok(())
266    }
267}
268
269fn repair_tail(path: &Path) -> Result<()> {
270    let content = match fs::read(path) {
271        Ok(content) => content,
272        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
273        Err(e) => return Err(Error::io("read", path, e)),
274    };
275    if content.is_empty() || content.ends_with(b"\n") {
276        return Ok(());
277    }
278    let keep = content
279        .iter()
280        .rposition(|b| *b == b'\n')
281        .map(|position| position + 1)
282        .unwrap_or(0);
283    let tail = &content[keep..];
284    if parse_record(tail).is_ok() {
285        let mut file = OpenOptions::new()
286            .append(true)
287            .open(path)
288            .map_err(|e| Error::io("open", path, e))?;
289        file.write_all(b"\n")
290            .map_err(|e| Error::io("append", path, e))?;
291    } else {
292        let file = OpenOptions::new()
293            .write(true)
294            .open(path)
295            .map_err(|e| Error::io("open", path, e))?;
296        file.set_len(keep as u64)
297            .map_err(|e| Error::io("truncate", path, e))?;
298    }
299    Ok(())
300}
301
302/// One shard's records plus reader warnings.
303#[derive(Debug, Clone)]
304pub struct Shard {
305    /// The records, in append order.
306    pub records: Vec<Record>,
307    /// Non-fatal reader findings, such as a discarded torn tail.
308    pub warnings: Vec<String>,
309}
310
311fn parse_record(line: &[u8]) -> std::result::Result<Record, String> {
312    let text = std::str::from_utf8(line)
313        .map_err(|e| format!("invalid UTF-8 at byte {}", e.valid_up_to()))?;
314    if text.ends_with('\r') {
315        return Err("carriage return before line ending, shards are LF only".to_string());
316    }
317    let record: Record = serde_json::from_str(text).map_err(|e| e.to_string())?;
318    record.validate()?;
319    Ok(record)
320}
321
322/// Reads every record from a shard in order.
323///
324/// The shard is read as bytes and split on LF, then each line decodes
325/// on its own, so damage to one line never hides the rest. A malformed
326/// complete line, invalid UTF-8, an unknown field, an unrecognized
327/// schema value, a carriage return, or a cross-field invariant
328/// violation fails the whole read and names the line. The one
329/// tolerated defect is a torn final line without a trailing LF, the
330/// footprint of a killed run, whether it breaks mid-record or
331/// mid-character. It is dropped with a warning, and the next run
332/// re-records the affected source.
333pub fn read_shard(path: &Path) -> Result<Shard> {
334    let content = fs::read(path).map_err(|e| Error::io("read", path, e))?;
335    let mut records = Vec::new();
336    let mut warnings = Vec::new();
337    if content.is_empty() {
338        return Ok(Shard { records, warnings });
339    }
340    let ends_with_lf = content.ends_with(b"\n");
341    let mut lines: Vec<&[u8]> = content.split(|byte| *byte == b'\n').collect();
342    if ends_with_lf {
343        lines.pop();
344    }
345    let count = lines.len();
346    for (index, line) in lines.iter().enumerate() {
347        let number = index + 1;
348        let partial = !ends_with_lf && number == count;
349        match parse_record(line) {
350            Ok(record) => records.push(record),
351            Err(_) if partial => {
352                warnings.push(format!("discarded torn final line {number}"));
353            }
354            Err(message) => {
355                return Err(Error::Manifest {
356                    path: path.to_path_buf(),
357                    line: number,
358                    message,
359                });
360            }
361        }
362    }
363    Ok(Shard { records, warnings })
364}
365
366/// Reads a shard with no torn-line tolerance.
367///
368/// The producer's own resume path tolerates one torn final line, the
369/// footprint of a killed run. A bundle must not: a torn tail means an
370/// interrupted run and a possibly incomplete division, so packaging
371/// and receiving both refuse it. Every line must parse as a complete
372/// record and the shard must end with a newline, the clean-write
373/// property bundle@1 declares.
374pub fn read_shard_strict(path: &Path) -> Result<Vec<Record>> {
375    let bytes = fs::read(path).map_err(|e| Error::io("read", path, e))?;
376    if !bytes.is_empty() && !bytes.ends_with(b"\n") {
377        return Err(Error::Manifest {
378            path: path.to_path_buf(),
379            line: 0,
380            message: "shard does not end with a newline, resume the run to complete the division"
381                .to_string(),
382        });
383    }
384    let shard = read_shard(path)?;
385    if let Some(warning) = shard.warnings.first() {
386        return Err(Error::Manifest {
387            path: path.to_path_buf(),
388            line: 0,
389            message: format!(
390                "shard has a torn final line ({warning}), resume the run to complete the division"
391            ),
392        });
393    }
394    Ok(shard.records)
395}
396
397#[cfg(test)]
398mod tests {
399    use super::*;
400
401    fn sample_record() -> Record {
402        Record {
403            schema: ManifestSchema,
404            source_path: "docs/a.txt".to_string(),
405            source_hash: "aa".to_string(),
406            source_size: 5,
407            declared_format: Some("text".to_string()),
408            detected_format: "text".to_string(),
409            format_mismatch: false,
410            status: Status::Converted,
411            text_path: Some("docs/a.txt.txt".to_string()),
412            text_hash: Some("bb".to_string()),
413            artifact_kind: Some(ArtifactKind::Text),
414            converter_id: Some("text-passthrough".to_string()),
415            converter_version: Some("1.0.0".to_string()),
416            tool_version: None,
417            rules_version: "1".to_string(),
418            media: None,
419            parent_source: None,
420            dedup_of: None,
421            warnings: vec!["stripped leading byte order mark".to_string()],
422            error: None,
423            duration_ms: Some(3),
424        }
425    }
426
427    #[test]
428    fn record_roundtrips_through_json() {
429        let record = sample_record();
430        let json = serde_json::to_string(&record).unwrap();
431        let back: Record = serde_json::from_str(&json).unwrap();
432        assert_eq!(record, back);
433    }
434
435    #[test]
436    fn schema_is_the_first_field() {
437        let json = serde_json::to_string(&sample_record()).unwrap();
438        assert!(json.starts_with(r#"{"schema":"text-mirror/manifest@1""#));
439    }
440
441    #[test]
442    fn unknown_schema_values_are_rejected() {
443        let json = serde_json::to_string(&sample_record()).unwrap();
444        let bumped = json.replace("text-mirror/manifest@1", "text-mirror/manifest@2");
445        let result: std::result::Result<Record, _> = serde_json::from_str(&bumped);
446        let message = result.unwrap_err().to_string();
447        assert!(message.contains("unrecognized schema"));
448    }
449
450    #[test]
451    fn unknown_fields_are_rejected() {
452        let json = serde_json::to_string(&sample_record()).unwrap();
453        let extended = json.replacen('{', r#"{"surprise":true,"#, 1);
454        let result: std::result::Result<Record, _> = serde_json::from_str(&extended);
455        assert!(result.is_err());
456    }
457
458    #[test]
459    fn writer_appends_one_lf_terminated_line_per_record() {
460        let dir = tempfile::tempdir().unwrap();
461        let path = dir.path().join("division.jsonl");
462        let mut writer = ManifestWriter::open(&path).unwrap();
463        writer.append(&sample_record()).unwrap();
464        let mut second = sample_record();
465        second.source_path = "docs/b.txt".to_string();
466        writer.append(&second).unwrap();
467        drop(writer);
468
469        let raw = std::fs::read_to_string(&path).unwrap();
470        assert_eq!(raw.lines().count(), 2);
471        assert!(raw.ends_with('\n'));
472        assert!(!raw.contains('\r'));
473
474        let shard = read_shard(&path).unwrap();
475        assert!(shard.warnings.is_empty());
476        assert_eq!(shard.records.len(), 2);
477        assert_eq!(shard.records[0], sample_record());
478        assert_eq!(shard.records[1].source_path, "docs/b.txt");
479    }
480
481    #[test]
482    fn read_shard_names_the_bad_line() {
483        let dir = tempfile::tempdir().unwrap();
484        let path = dir.path().join("division.jsonl");
485        let good = serde_json::to_string(&sample_record()).unwrap();
486        std::fs::write(&path, format!("{good}\nnot json\n")).unwrap();
487        let err = read_shard(&path).unwrap_err();
488        assert!(err.to_string().contains("line 2"));
489    }
490
491    #[test]
492    fn read_shard_tolerates_only_a_torn_final_line() {
493        let dir = tempfile::tempdir().unwrap();
494        let path = dir.path().join("division.jsonl");
495        let good = serde_json::to_string(&sample_record()).unwrap();
496        let torn = &good[..good.len() / 2];
497        std::fs::write(&path, format!("{good}\n{torn}")).unwrap();
498
499        let shard = read_shard(&path).unwrap();
500        assert_eq!(shard.records.len(), 1);
501        assert_eq!(shard.warnings.len(), 1);
502        assert!(shard.warnings[0].contains("line 2"));
503
504        // The same damage on a non-final line still fails the read.
505        std::fs::write(&path, format!("{torn}\n{good}\n")).unwrap();
506        let err = read_shard(&path).unwrap_err();
507        assert!(err.to_string().contains("line 1"));
508    }
509
510    #[test]
511    fn read_shard_accepts_a_complete_final_record_without_lf() {
512        let dir = tempfile::tempdir().unwrap();
513        let path = dir.path().join("division.jsonl");
514        let good = serde_json::to_string(&sample_record()).unwrap();
515        std::fs::write(&path, &good).unwrap();
516        let shard = read_shard(&path).unwrap();
517        assert_eq!(shard.records.len(), 1);
518        assert!(shard.warnings.is_empty());
519    }
520
521    #[test]
522    fn read_shard_strict_rejects_an_unterminated_final_record() {
523        // The tolerant reader above accepts this shape for the
524        // producer's resume path. The strict reader enforces the
525        // clean-write property itself, with no caller byte checks.
526        let dir = tempfile::tempdir().unwrap();
527        let path = dir.path().join("division.jsonl");
528        let good = serde_json::to_string(&sample_record()).unwrap();
529        std::fs::write(&path, &good).unwrap();
530        let err = read_shard_strict(&path).unwrap_err();
531        assert!(
532            err.to_string().contains("does not end with a newline"),
533            "{err}"
534        );
535
536        std::fs::write(&path, format!("{good}\n")).unwrap();
537        assert_eq!(read_shard_strict(&path).unwrap().len(), 1);
538    }
539
540    #[test]
541    fn writer_finishes_a_final_line_that_lacks_lf_before_appending() {
542        let dir = tempfile::tempdir().unwrap();
543        let path = dir.path().join("division.jsonl");
544        let good = serde_json::to_string(&sample_record()).unwrap();
545        std::fs::write(&path, &good).unwrap();
546
547        let mut writer = ManifestWriter::open(&path).unwrap();
548        let mut second = sample_record();
549        second.source_path = "docs/b.txt".to_string();
550        writer.append(&second).unwrap();
551        drop(writer);
552
553        let shard = read_shard(&path).unwrap();
554        assert_eq!(shard.records.len(), 2);
555        assert!(shard.warnings.is_empty());
556    }
557
558    #[test]
559    fn writer_drops_a_torn_tail_before_appending() {
560        let dir = tempfile::tempdir().unwrap();
561        let path = dir.path().join("division.jsonl");
562        let good = serde_json::to_string(&sample_record()).unwrap();
563        let torn = &good[..good.len() / 2];
564        std::fs::write(&path, format!("{good}\n{torn}")).unwrap();
565
566        let mut writer = ManifestWriter::open(&path).unwrap();
567        let mut second = sample_record();
568        second.source_path = "docs/b.txt".to_string();
569        writer.append(&second).unwrap();
570        drop(writer);
571
572        let shard = read_shard(&path).unwrap();
573        assert!(shard.warnings.is_empty());
574        assert_eq!(shard.records.len(), 2);
575        assert_eq!(shard.records[1].source_path, "docs/b.txt");
576    }
577
578    #[test]
579    fn read_shard_tolerates_a_tail_torn_inside_a_multibyte_character() {
580        let dir = tempfile::tempdir().unwrap();
581        let path = dir.path().join("division.jsonl");
582        let mut accented = sample_record();
583        accented.source_path = "R\u{e9}sum\u{e9}.txt".to_string();
584        let line = serde_json::to_string(&accented).unwrap();
585        let bytes = line.as_bytes();
586
587        // Cut immediately after the 0xC3 lead byte of the first
588        // multibyte character, the footprint of a killed mid-write.
589        let lead = bytes.iter().position(|b| *b == 0xC3).unwrap();
590        let good = serde_json::to_string(&sample_record()).unwrap();
591        let mut damaged = format!("{good}\n").into_bytes();
592        damaged.extend_from_slice(&bytes[..=lead]);
593        std::fs::write(&path, &damaged).unwrap();
594
595        let shard = read_shard(&path).unwrap();
596        assert_eq!(shard.records.len(), 1);
597        assert_eq!(shard.warnings.len(), 1);
598        assert!(shard.warnings[0].contains("line 2"));
599
600        // The writer repairs the same tail, and appends land clean.
601        let mut writer = ManifestWriter::open(&path).unwrap();
602        writer.append(&accented).unwrap();
603        drop(writer);
604        let shard = read_shard(&path).unwrap();
605        assert!(shard.warnings.is_empty());
606        assert_eq!(shard.records.len(), 2);
607        assert_eq!(shard.records[1].source_path, "R\u{e9}sum\u{e9}.txt");
608    }
609
610    #[test]
611    fn read_shard_rejects_invalid_utf8_on_a_complete_line() {
612        let dir = tempfile::tempdir().unwrap();
613        let path = dir.path().join("division.jsonl");
614        let good = serde_json::to_string(&sample_record()).unwrap();
615        let mut damaged = b"\xc3\x28 not a record\n".to_vec();
616        damaged.extend_from_slice(format!("{good}\n").as_bytes());
617        std::fs::write(&path, &damaged).unwrap();
618
619        let err = read_shard(&path).unwrap_err();
620        let message = err.to_string();
621        assert!(message.contains("line 1"));
622        assert!(message.contains("invalid UTF-8"));
623    }
624
625    #[test]
626    fn read_shard_rejects_carriage_returns() {
627        let dir = tempfile::tempdir().unwrap();
628        let path = dir.path().join("division.jsonl");
629        let good = serde_json::to_string(&sample_record()).unwrap();
630        std::fs::write(&path, format!("{good}\r\n")).unwrap();
631        let err = read_shard(&path).unwrap_err();
632        let message = err.to_string();
633        assert!(message.contains("line 1"));
634        assert!(message.contains("carriage return"));
635    }
636
637    #[test]
638    fn read_shard_rejects_cross_field_invariant_violations() {
639        let dir = tempfile::tempdir().unwrap();
640        let path = dir.path().join("division.jsonl");
641        let good = serde_json::to_string(&sample_record()).unwrap();
642
643        // A converted record with no text fields is a false success.
644        let mut bare = sample_record();
645        bare.text_path = None;
646        bare.text_hash = None;
647        let bad = serde_json::to_string(&bare).unwrap();
648        std::fs::write(&path, format!("{good}\n{bad}\n")).unwrap();
649        let err = read_shard(&path).unwrap_err();
650        let message = err.to_string();
651        assert!(message.contains("line 2"));
652        assert!(message.contains("requires text_path"));
653
654        // A failed record must carry a reason and no text fields.
655        let mut failed = sample_record();
656        failed.status = Status::Failed;
657        failed.error = Some("invalid_utf8: invalid UTF-8 at byte 0".to_string());
658        let bad = serde_json::to_string(&failed).unwrap();
659        std::fs::write(&path, format!("{bad}\n")).unwrap();
660        assert!(read_shard(&path).is_err());
661
662        // A dedup record must name its canonical source.
663        let mut dedup = sample_record();
664        dedup.status = Status::Dedup;
665        let bad = serde_json::to_string(&dedup).unwrap();
666        std::fs::write(&path, format!("{bad}\n")).unwrap();
667        assert!(read_shard(&path).is_err());
668    }
669
670    #[test]
671    fn error_is_forbidden_on_success_statuses() {
672        for status in [Status::Converted, Status::SkippedUnchanged, Status::Dedup] {
673            let mut record = sample_record();
674            record.status = status;
675            if status == Status::Dedup {
676                record.dedup_of = Some("other.txt".to_string());
677            }
678            record.error = Some("stray reason".to_string());
679            let message = record.validate().unwrap_err();
680            assert!(message.contains("forbids error"), "{message}");
681        }
682
683        // Unsupported may carry the declared capability gap.
684        let mut unsupported = sample_record();
685        unsupported.status = Status::Unsupported;
686        unsupported.text_path = None;
687        unsupported.text_hash = None;
688        unsupported.error = Some("hidden-visibility-unresolved".to_string());
689        unsupported.validate().unwrap();
690    }
691}