1use std::fs::{self, File, OpenOptions};
17use std::io::Write;
18use std::path::{Path, PathBuf};
19
20use serde::de::Error as _;
21use serde::{Deserialize, Deserializer, Serialize, Serializer};
22
23use crate::{Error, Result};
24
25pub const MANIFEST_SCHEMA: &str = "text-mirror/manifest@1";
27
28#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
34pub struct ManifestSchema;
35
36impl Serialize for ManifestSchema {
37 fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
38 serializer.serialize_str(MANIFEST_SCHEMA)
39 }
40}
41
42impl<'de> Deserialize<'de> for ManifestSchema {
43 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
44 let value = String::deserialize(deserializer)?;
45 if value == MANIFEST_SCHEMA {
46 Ok(ManifestSchema)
47 } else {
48 Err(D::Error::custom(format!(
49 "unrecognized schema {value:?}, this consumer accepts only {MANIFEST_SCHEMA:?}"
50 )))
51 }
52 }
53}
54
55#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
57#[serde(rename_all = "snake_case")]
58pub enum Status {
59 Converted,
61 Failed,
63 Unsupported,
65 SkippedUnchanged,
67 Dedup,
70}
71
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
74#[serde(rename_all = "snake_case")]
75pub enum ArtifactKind {
76 Text,
78 Transcript,
80 Ocr,
82 Mixed,
84}
85
86#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
88#[serde(deny_unknown_fields)]
89pub struct Media {
90 #[serde(skip_serializing_if = "Option::is_none", default)]
92 pub duration_seconds: Option<f64>,
93 #[serde(skip_serializing_if = "Option::is_none", default)]
95 pub language: Option<String>,
96 #[serde(skip_serializing_if = "Option::is_none", default)]
98 pub model_id: Option<String>,
99 #[serde(skip_serializing_if = "Option::is_none", default)]
101 pub model_hash: Option<String>,
102 #[serde(skip_serializing_if = "Option::is_none", default)]
104 pub decode_options_hash: Option<String>,
105 #[serde(skip_serializing_if = "Option::is_none", default)]
107 pub speaker_count: Option<u32>,
108}
109
110#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
115#[serde(deny_unknown_fields)]
116pub struct Record {
117 pub schema: ManifestSchema,
119 pub source_path: String,
121 pub source_hash: String,
123 pub source_size: u64,
125 #[serde(skip_serializing_if = "Option::is_none", default)]
127 pub declared_format: Option<String>,
128 pub detected_format: String,
130 pub format_mismatch: bool,
132 pub status: Status,
134 #[serde(skip_serializing_if = "Option::is_none", default)]
137 pub text_path: Option<String>,
138 #[serde(skip_serializing_if = "Option::is_none", default)]
140 pub text_hash: Option<String>,
141 #[serde(skip_serializing_if = "Option::is_none", default)]
143 pub artifact_kind: Option<ArtifactKind>,
144 #[serde(skip_serializing_if = "Option::is_none", default)]
146 pub converter_id: Option<String>,
147 #[serde(skip_serializing_if = "Option::is_none", default)]
149 pub converter_version: Option<String>,
150 #[serde(skip_serializing_if = "Option::is_none", default)]
152 pub tool_version: Option<String>,
153 pub rules_version: String,
155 #[serde(skip_serializing_if = "Option::is_none", default)]
157 pub media: Option<Media>,
158 #[serde(skip_serializing_if = "Option::is_none", default)]
160 pub parent_source: Option<String>,
161 #[serde(skip_serializing_if = "Option::is_none", default)]
163 pub dedup_of: Option<String>,
164 #[serde(skip_serializing_if = "Vec::is_empty", default)]
166 pub warnings: Vec<String>,
167 #[serde(skip_serializing_if = "Option::is_none", default)]
171 pub error: Option<String>,
172 #[serde(skip_serializing_if = "Option::is_none", default)]
174 pub duration_ms: Option<u64>,
175}
176
177fn status_name(status: Status) -> &'static str {
178 match status {
179 Status::Converted => "converted",
180 Status::Failed => "failed",
181 Status::Unsupported => "unsupported",
182 Status::SkippedUnchanged => "skipped_unchanged",
183 Status::Dedup => "dedup",
184 }
185}
186
187impl Record {
188 pub fn validate(&self) -> std::result::Result<(), String> {
197 let name = status_name(self.status);
198 match self.status {
199 Status::Converted | Status::SkippedUnchanged | Status::Dedup => {
200 if self.text_path.is_none() || self.text_hash.is_none() {
201 return Err(format!("status {name} requires text_path and text_hash"));
202 }
203 if self.error.is_some() {
204 return Err(format!("status {name} forbids error"));
205 }
206 }
207 Status::Failed | Status::Unsupported => {
208 if self.text_path.is_some() || self.text_hash.is_some() {
209 return Err(format!("status {name} forbids text_path and text_hash"));
210 }
211 }
212 }
213 if self.status == Status::Dedup && self.dedup_of.is_none() {
214 return Err("status dedup requires dedup_of".to_string());
215 }
216 if self.status == Status::Failed && self.error.is_none() {
217 return Err("status failed requires error".to_string());
218 }
219 Ok(())
220 }
221}
222
223pub struct ManifestWriter {
225 path: PathBuf,
226 file: File,
227}
228
229impl ManifestWriter {
230 pub fn open(path: &Path) -> Result<Self> {
237 if let Some(dir) = path.parent() {
238 fs::create_dir_all(dir).map_err(|e| Error::io("create_dir", dir, e))?;
239 }
240 repair_tail(path)?;
241 let file = OpenOptions::new()
242 .create(true)
243 .append(true)
244 .open(path)
245 .map_err(|e| Error::io("open", path, e))?;
246 Ok(ManifestWriter {
247 path: path.to_path_buf(),
248 file,
249 })
250 }
251
252 pub fn append(&mut self, record: &Record) -> Result<()> {
254 let mut line = serde_json::to_string(record).map_err(|e| Error::Encode {
255 path: self.path.clone(),
256 message: e.to_string(),
257 })?;
258 line.push('\n');
259 self.file
260 .write_all(line.as_bytes())
261 .map_err(|e| Error::io("append", &self.path, e))?;
262 self.file
263 .flush()
264 .map_err(|e| Error::io("flush", &self.path, e))?;
265 Ok(())
266 }
267}
268
269fn repair_tail(path: &Path) -> Result<()> {
270 let content = match fs::read(path) {
271 Ok(content) => content,
272 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
273 Err(e) => return Err(Error::io("read", path, e)),
274 };
275 if content.is_empty() || content.ends_with(b"\n") {
276 return Ok(());
277 }
278 let keep = content
279 .iter()
280 .rposition(|b| *b == b'\n')
281 .map(|position| position + 1)
282 .unwrap_or(0);
283 let tail = &content[keep..];
284 if parse_record(tail).is_ok() {
285 let mut file = OpenOptions::new()
286 .append(true)
287 .open(path)
288 .map_err(|e| Error::io("open", path, e))?;
289 file.write_all(b"\n")
290 .map_err(|e| Error::io("append", path, e))?;
291 } else {
292 let file = OpenOptions::new()
293 .write(true)
294 .open(path)
295 .map_err(|e| Error::io("open", path, e))?;
296 file.set_len(keep as u64)
297 .map_err(|e| Error::io("truncate", path, e))?;
298 }
299 Ok(())
300}
301
302#[derive(Debug, Clone)]
304pub struct Shard {
305 pub records: Vec<Record>,
307 pub warnings: Vec<String>,
309}
310
311fn parse_record(line: &[u8]) -> std::result::Result<Record, String> {
312 let text = std::str::from_utf8(line)
313 .map_err(|e| format!("invalid UTF-8 at byte {}", e.valid_up_to()))?;
314 if text.ends_with('\r') {
315 return Err("carriage return before line ending, shards are LF only".to_string());
316 }
317 let record: Record = serde_json::from_str(text).map_err(|e| e.to_string())?;
318 record.validate()?;
319 Ok(record)
320}
321
322pub fn read_shard(path: &Path) -> Result<Shard> {
334 let content = fs::read(path).map_err(|e| Error::io("read", path, e))?;
335 let mut records = Vec::new();
336 let mut warnings = Vec::new();
337 if content.is_empty() {
338 return Ok(Shard { records, warnings });
339 }
340 let ends_with_lf = content.ends_with(b"\n");
341 let mut lines: Vec<&[u8]> = content.split(|byte| *byte == b'\n').collect();
342 if ends_with_lf {
343 lines.pop();
344 }
345 let count = lines.len();
346 for (index, line) in lines.iter().enumerate() {
347 let number = index + 1;
348 let partial = !ends_with_lf && number == count;
349 match parse_record(line) {
350 Ok(record) => records.push(record),
351 Err(_) if partial => {
352 warnings.push(format!("discarded torn final line {number}"));
353 }
354 Err(message) => {
355 return Err(Error::Manifest {
356 path: path.to_path_buf(),
357 line: number,
358 message,
359 });
360 }
361 }
362 }
363 Ok(Shard { records, warnings })
364}
365
366pub fn read_shard_strict(path: &Path) -> Result<Vec<Record>> {
375 let bytes = fs::read(path).map_err(|e| Error::io("read", path, e))?;
376 if !bytes.is_empty() && !bytes.ends_with(b"\n") {
377 return Err(Error::Manifest {
378 path: path.to_path_buf(),
379 line: 0,
380 message: "shard does not end with a newline, resume the run to complete the division"
381 .to_string(),
382 });
383 }
384 let shard = read_shard(path)?;
385 if let Some(warning) = shard.warnings.first() {
386 return Err(Error::Manifest {
387 path: path.to_path_buf(),
388 line: 0,
389 message: format!(
390 "shard has a torn final line ({warning}), resume the run to complete the division"
391 ),
392 });
393 }
394 Ok(shard.records)
395}
396
397#[cfg(test)]
398mod tests {
399 use super::*;
400
401 fn sample_record() -> Record {
402 Record {
403 schema: ManifestSchema,
404 source_path: "docs/a.txt".to_string(),
405 source_hash: "aa".to_string(),
406 source_size: 5,
407 declared_format: Some("text".to_string()),
408 detected_format: "text".to_string(),
409 format_mismatch: false,
410 status: Status::Converted,
411 text_path: Some("docs/a.txt.txt".to_string()),
412 text_hash: Some("bb".to_string()),
413 artifact_kind: Some(ArtifactKind::Text),
414 converter_id: Some("text-passthrough".to_string()),
415 converter_version: Some("1.0.0".to_string()),
416 tool_version: None,
417 rules_version: "1".to_string(),
418 media: None,
419 parent_source: None,
420 dedup_of: None,
421 warnings: vec!["stripped leading byte order mark".to_string()],
422 error: None,
423 duration_ms: Some(3),
424 }
425 }
426
427 #[test]
428 fn record_roundtrips_through_json() {
429 let record = sample_record();
430 let json = serde_json::to_string(&record).unwrap();
431 let back: Record = serde_json::from_str(&json).unwrap();
432 assert_eq!(record, back);
433 }
434
435 #[test]
436 fn schema_is_the_first_field() {
437 let json = serde_json::to_string(&sample_record()).unwrap();
438 assert!(json.starts_with(r#"{"schema":"text-mirror/manifest@1""#));
439 }
440
441 #[test]
442 fn unknown_schema_values_are_rejected() {
443 let json = serde_json::to_string(&sample_record()).unwrap();
444 let bumped = json.replace("text-mirror/manifest@1", "text-mirror/manifest@2");
445 let result: std::result::Result<Record, _> = serde_json::from_str(&bumped);
446 let message = result.unwrap_err().to_string();
447 assert!(message.contains("unrecognized schema"));
448 }
449
450 #[test]
451 fn unknown_fields_are_rejected() {
452 let json = serde_json::to_string(&sample_record()).unwrap();
453 let extended = json.replacen('{', r#"{"surprise":true,"#, 1);
454 let result: std::result::Result<Record, _> = serde_json::from_str(&extended);
455 assert!(result.is_err());
456 }
457
458 #[test]
459 fn writer_appends_one_lf_terminated_line_per_record() {
460 let dir = tempfile::tempdir().unwrap();
461 let path = dir.path().join("division.jsonl");
462 let mut writer = ManifestWriter::open(&path).unwrap();
463 writer.append(&sample_record()).unwrap();
464 let mut second = sample_record();
465 second.source_path = "docs/b.txt".to_string();
466 writer.append(&second).unwrap();
467 drop(writer);
468
469 let raw = std::fs::read_to_string(&path).unwrap();
470 assert_eq!(raw.lines().count(), 2);
471 assert!(raw.ends_with('\n'));
472 assert!(!raw.contains('\r'));
473
474 let shard = read_shard(&path).unwrap();
475 assert!(shard.warnings.is_empty());
476 assert_eq!(shard.records.len(), 2);
477 assert_eq!(shard.records[0], sample_record());
478 assert_eq!(shard.records[1].source_path, "docs/b.txt");
479 }
480
481 #[test]
482 fn read_shard_names_the_bad_line() {
483 let dir = tempfile::tempdir().unwrap();
484 let path = dir.path().join("division.jsonl");
485 let good = serde_json::to_string(&sample_record()).unwrap();
486 std::fs::write(&path, format!("{good}\nnot json\n")).unwrap();
487 let err = read_shard(&path).unwrap_err();
488 assert!(err.to_string().contains("line 2"));
489 }
490
491 #[test]
492 fn read_shard_tolerates_only_a_torn_final_line() {
493 let dir = tempfile::tempdir().unwrap();
494 let path = dir.path().join("division.jsonl");
495 let good = serde_json::to_string(&sample_record()).unwrap();
496 let torn = &good[..good.len() / 2];
497 std::fs::write(&path, format!("{good}\n{torn}")).unwrap();
498
499 let shard = read_shard(&path).unwrap();
500 assert_eq!(shard.records.len(), 1);
501 assert_eq!(shard.warnings.len(), 1);
502 assert!(shard.warnings[0].contains("line 2"));
503
504 std::fs::write(&path, format!("{torn}\n{good}\n")).unwrap();
506 let err = read_shard(&path).unwrap_err();
507 assert!(err.to_string().contains("line 1"));
508 }
509
510 #[test]
511 fn read_shard_accepts_a_complete_final_record_without_lf() {
512 let dir = tempfile::tempdir().unwrap();
513 let path = dir.path().join("division.jsonl");
514 let good = serde_json::to_string(&sample_record()).unwrap();
515 std::fs::write(&path, &good).unwrap();
516 let shard = read_shard(&path).unwrap();
517 assert_eq!(shard.records.len(), 1);
518 assert!(shard.warnings.is_empty());
519 }
520
521 #[test]
522 fn read_shard_strict_rejects_an_unterminated_final_record() {
523 let dir = tempfile::tempdir().unwrap();
527 let path = dir.path().join("division.jsonl");
528 let good = serde_json::to_string(&sample_record()).unwrap();
529 std::fs::write(&path, &good).unwrap();
530 let err = read_shard_strict(&path).unwrap_err();
531 assert!(
532 err.to_string().contains("does not end with a newline"),
533 "{err}"
534 );
535
536 std::fs::write(&path, format!("{good}\n")).unwrap();
537 assert_eq!(read_shard_strict(&path).unwrap().len(), 1);
538 }
539
540 #[test]
541 fn writer_finishes_a_final_line_that_lacks_lf_before_appending() {
542 let dir = tempfile::tempdir().unwrap();
543 let path = dir.path().join("division.jsonl");
544 let good = serde_json::to_string(&sample_record()).unwrap();
545 std::fs::write(&path, &good).unwrap();
546
547 let mut writer = ManifestWriter::open(&path).unwrap();
548 let mut second = sample_record();
549 second.source_path = "docs/b.txt".to_string();
550 writer.append(&second).unwrap();
551 drop(writer);
552
553 let shard = read_shard(&path).unwrap();
554 assert_eq!(shard.records.len(), 2);
555 assert!(shard.warnings.is_empty());
556 }
557
558 #[test]
559 fn writer_drops_a_torn_tail_before_appending() {
560 let dir = tempfile::tempdir().unwrap();
561 let path = dir.path().join("division.jsonl");
562 let good = serde_json::to_string(&sample_record()).unwrap();
563 let torn = &good[..good.len() / 2];
564 std::fs::write(&path, format!("{good}\n{torn}")).unwrap();
565
566 let mut writer = ManifestWriter::open(&path).unwrap();
567 let mut second = sample_record();
568 second.source_path = "docs/b.txt".to_string();
569 writer.append(&second).unwrap();
570 drop(writer);
571
572 let shard = read_shard(&path).unwrap();
573 assert!(shard.warnings.is_empty());
574 assert_eq!(shard.records.len(), 2);
575 assert_eq!(shard.records[1].source_path, "docs/b.txt");
576 }
577
578 #[test]
579 fn read_shard_tolerates_a_tail_torn_inside_a_multibyte_character() {
580 let dir = tempfile::tempdir().unwrap();
581 let path = dir.path().join("division.jsonl");
582 let mut accented = sample_record();
583 accented.source_path = "R\u{e9}sum\u{e9}.txt".to_string();
584 let line = serde_json::to_string(&accented).unwrap();
585 let bytes = line.as_bytes();
586
587 let lead = bytes.iter().position(|b| *b == 0xC3).unwrap();
590 let good = serde_json::to_string(&sample_record()).unwrap();
591 let mut damaged = format!("{good}\n").into_bytes();
592 damaged.extend_from_slice(&bytes[..=lead]);
593 std::fs::write(&path, &damaged).unwrap();
594
595 let shard = read_shard(&path).unwrap();
596 assert_eq!(shard.records.len(), 1);
597 assert_eq!(shard.warnings.len(), 1);
598 assert!(shard.warnings[0].contains("line 2"));
599
600 let mut writer = ManifestWriter::open(&path).unwrap();
602 writer.append(&accented).unwrap();
603 drop(writer);
604 let shard = read_shard(&path).unwrap();
605 assert!(shard.warnings.is_empty());
606 assert_eq!(shard.records.len(), 2);
607 assert_eq!(shard.records[1].source_path, "R\u{e9}sum\u{e9}.txt");
608 }
609
610 #[test]
611 fn read_shard_rejects_invalid_utf8_on_a_complete_line() {
612 let dir = tempfile::tempdir().unwrap();
613 let path = dir.path().join("division.jsonl");
614 let good = serde_json::to_string(&sample_record()).unwrap();
615 let mut damaged = b"\xc3\x28 not a record\n".to_vec();
616 damaged.extend_from_slice(format!("{good}\n").as_bytes());
617 std::fs::write(&path, &damaged).unwrap();
618
619 let err = read_shard(&path).unwrap_err();
620 let message = err.to_string();
621 assert!(message.contains("line 1"));
622 assert!(message.contains("invalid UTF-8"));
623 }
624
625 #[test]
626 fn read_shard_rejects_carriage_returns() {
627 let dir = tempfile::tempdir().unwrap();
628 let path = dir.path().join("division.jsonl");
629 let good = serde_json::to_string(&sample_record()).unwrap();
630 std::fs::write(&path, format!("{good}\r\n")).unwrap();
631 let err = read_shard(&path).unwrap_err();
632 let message = err.to_string();
633 assert!(message.contains("line 1"));
634 assert!(message.contains("carriage return"));
635 }
636
637 #[test]
638 fn read_shard_rejects_cross_field_invariant_violations() {
639 let dir = tempfile::tempdir().unwrap();
640 let path = dir.path().join("division.jsonl");
641 let good = serde_json::to_string(&sample_record()).unwrap();
642
643 let mut bare = sample_record();
645 bare.text_path = None;
646 bare.text_hash = None;
647 let bad = serde_json::to_string(&bare).unwrap();
648 std::fs::write(&path, format!("{good}\n{bad}\n")).unwrap();
649 let err = read_shard(&path).unwrap_err();
650 let message = err.to_string();
651 assert!(message.contains("line 2"));
652 assert!(message.contains("requires text_path"));
653
654 let mut failed = sample_record();
656 failed.status = Status::Failed;
657 failed.error = Some("invalid_utf8: invalid UTF-8 at byte 0".to_string());
658 let bad = serde_json::to_string(&failed).unwrap();
659 std::fs::write(&path, format!("{bad}\n")).unwrap();
660 assert!(read_shard(&path).is_err());
661
662 let mut dedup = sample_record();
664 dedup.status = Status::Dedup;
665 let bad = serde_json::to_string(&dedup).unwrap();
666 std::fs::write(&path, format!("{bad}\n")).unwrap();
667 assert!(read_shard(&path).is_err());
668 }
669
670 #[test]
671 fn error_is_forbidden_on_success_statuses() {
672 for status in [Status::Converted, Status::SkippedUnchanged, Status::Dedup] {
673 let mut record = sample_record();
674 record.status = status;
675 if status == Status::Dedup {
676 record.dedup_of = Some("other.txt".to_string());
677 }
678 record.error = Some("stray reason".to_string());
679 let message = record.validate().unwrap_err();
680 assert!(message.contains("forbids error"), "{message}");
681 }
682
683 let mut unsupported = sample_record();
685 unsupported.status = Status::Unsupported;
686 unsupported.text_path = None;
687 unsupported.text_hash = None;
688 unsupported.error = Some("hidden-visibility-unresolved".to_string());
689 unsupported.validate().unwrap();
690 }
691}