text_mirror/
segments.rs

1//! The `<source>.segments.jsonl` file beside each converted artifact.
2//!
3//! Segments map the content text back to source structure, out of
4//! band. The artifact text carries no marker strings. A boundary is a
5//! zero-width record naming the start of a page, sheet, or slide. A
6//! span covers a half-open byte range of the artifact and can mark
7//! content that was hidden in the source. Records are ordered by
8//! start offset, then end offset.
9
10use std::path::Path;
11
12use serde::de::Error as _;
13use serde::{Deserialize, Deserializer, Serialize, Serializer};
14
15use crate::mirror;
16use crate::{Error, Result};
17
18/// The schema identifier carried by every segments@1 record.
19pub const SEGMENTS_SCHEMA: &str = "text-mirror/segments@1";
20
21/// The offset unit every segments@1 record uses.
22pub const SEGMENTS_UNIT: &str = "utf8_bytes";
23
24/// The `schema` field of a [`Segment`]. Serializes to the literal
25/// `text-mirror/segments@1` and rejects every other value on read.
26#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
27pub struct SegmentsSchema;
28
29impl Serialize for SegmentsSchema {
30    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
31        serializer.serialize_str(SEGMENTS_SCHEMA)
32    }
33}
34
35impl<'de> Deserialize<'de> for SegmentsSchema {
36    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
37        let value = String::deserialize(deserializer)?;
38        if value == SEGMENTS_SCHEMA {
39            Ok(SegmentsSchema)
40        } else {
41            Err(D::Error::custom(format!(
42                "unrecognized schema {value:?}, this consumer accepts only {SEGMENTS_SCHEMA:?}"
43            )))
44        }
45    }
46}
47
48/// What a segments record is.
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
50#[serde(rename_all = "snake_case")]
51pub enum SegmentKind {
52    /// A content span over a byte range of the artifact.
53    Span,
54    /// Zero-width boundary at the start of a page.
55    Page,
56    /// Zero-width boundary at the start of a sheet.
57    Sheet,
58    /// Zero-width boundary at the start of a slide.
59    Slide,
60}
61
62/// One record of a segments file.
63#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
64#[serde(deny_unknown_fields)]
65pub struct Segment {
66    /// The literal string `text-mirror/segments@1`, always first.
67    pub schema: SegmentsSchema,
68    /// Record kind. Boundary kinds are zero-width.
69    pub kind: SegmentKind,
70    /// Start offset into the artifact, in UTF-8 bytes.
71    pub start: u64,
72    /// End offset, exclusive. Equal to `start` for a boundary.
73    pub end: u64,
74    /// The offset unit, the literal `utf8_bytes`.
75    pub unit: String,
76    /// The source structure a span maps to, such as `document`,
77    /// `sheet`, `row`, or `column`.
78    #[serde(skip_serializing_if = "Option::is_none", default)]
79    pub source: Option<String>,
80    /// Structure name, such as a sheet name.
81    #[serde(skip_serializing_if = "Option::is_none", default)]
82    pub name: Option<String>,
83    /// True when the covered content was hidden in the source.
84    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
85    pub hidden: bool,
86}
87
88impl Segment {
89    /// A content span over `start..end`.
90    pub fn span(start: usize, end: usize, source: &str) -> Segment {
91        Segment {
92            schema: SegmentsSchema,
93            kind: SegmentKind::Span,
94            start: start as u64,
95            end: end as u64,
96            unit: SEGMENTS_UNIT.to_string(),
97            source: Some(source.to_string()),
98            name: None,
99            hidden: false,
100        }
101    }
102
103    /// A zero-width boundary at `at`.
104    pub fn boundary(kind: SegmentKind, at: usize) -> Segment {
105        Segment {
106            schema: SegmentsSchema,
107            kind,
108            start: at as u64,
109            end: at as u64,
110            unit: SEGMENTS_UNIT.to_string(),
111            source: None,
112            name: None,
113            hidden: false,
114        }
115    }
116
117    /// The same segment with a name attached.
118    pub fn named(mut self, name: &str) -> Segment {
119        self.name = Some(name.to_string());
120        self
121    }
122
123    /// The same segment marked hidden.
124    pub fn hidden(mut self) -> Segment {
125        self.hidden = true;
126        self
127    }
128}
129
130/// Maps a source path to its segments file in the mirror, the full
131/// source name plus `.segments.jsonl`.
132pub fn segments_path(
133    mirror_root: &Path,
134    division: &str,
135    relative_source: &Path,
136) -> std::path::PathBuf {
137    let mut mapped = mirror::mirror_path(mirror_root, division, relative_source);
138    let mut name = mapped
139        .file_name()
140        .map(std::ffi::OsString::from)
141        .unwrap_or_default();
142    // mirror_path appended ".txt", swap that suffix for ".segments.jsonl"
143    let text_name = name.to_string_lossy().into_owned();
144    let base = text_name.strip_suffix(".txt").unwrap_or(&text_name);
145    name = std::ffi::OsString::from(format!("{base}.segments.jsonl"));
146    mapped.set_file_name(name);
147    mapped
148}
149
150/// Serializes segments to JSONL, one record per line, LF only.
151pub fn to_jsonl(segments: &[Segment]) -> Result<String> {
152    let mut out = String::new();
153    for segment in segments {
154        let line = serde_json::to_string(segment).map_err(|e| Error::Encode {
155            path: std::path::PathBuf::from("segments"),
156            message: e.to_string(),
157        })?;
158        out.push_str(&line);
159        out.push('\n');
160    }
161    Ok(out)
162}
163
164/// Parses a segments sidecar back into records. Any malformed line or
165/// unrecognized schema fails the whole parse.
166pub fn parse_jsonl(content: &str) -> std::result::Result<Vec<Segment>, String> {
167    content
168        .lines()
169        .enumerate()
170        .map(|(index, line)| {
171            serde_json::from_str(line).map_err(|e| format!("segments line {}: {e}", index + 1))
172        })
173        .collect()
174}
175
176/// Checks that segments address `text` cleanly.
177///
178/// Every offset must land on a UTF-8 character boundary within the
179/// text, spans must be half-open with `start <= end`, boundaries must
180/// be zero-width, and records must be ordered by start, then end.
181pub fn validate(segments: &[Segment], text: &str) -> std::result::Result<(), String> {
182    let len = text.len() as u64;
183    let mut previous = (0u64, 0u64);
184    for (index, segment) in segments.iter().enumerate() {
185        if segment.start > segment.end {
186            return Err(format!(
187                "segment {index}: start {} after end {}",
188                segment.start, segment.end
189            ));
190        }
191        if segment.end > len {
192            return Err(format!(
193                "segment {index}: end {} past text length {len}",
194                segment.end
195            ));
196        }
197        if segment.kind != SegmentKind::Span && segment.start != segment.end {
198            return Err(format!("segment {index}: boundary is not zero-width"));
199        }
200        for offset in [segment.start, segment.end] {
201            if !text.is_char_boundary(offset as usize) {
202                return Err(format!(
203                    "segment {index}: offset {offset} splits a character"
204                ));
205            }
206        }
207        if (segment.start, segment.end) < previous {
208            return Err(format!("segment {index}: out of order"));
209        }
210        previous = (segment.start, segment.end);
211    }
212    Ok(())
213}
214
215#[cfg(test)]
216mod tests {
217    use super::*;
218
219    #[test]
220    fn segments_serialize_with_schema_first() {
221        let segment = Segment::span(0, 4, "document");
222        let json = serde_json::to_string(&segment).unwrap();
223        assert!(json.starts_with(r#"{"schema":"text-mirror/segments@1""#));
224        let back: Segment = serde_json::from_str(&json).unwrap();
225        assert_eq!(segment, back);
226    }
227
228    #[test]
229    fn unknown_segment_schema_is_rejected() {
230        let json = serde_json::to_string(&Segment::span(0, 1, "document")).unwrap();
231        let bumped = json.replace("segments@1", "segments@2");
232        assert!(serde_json::from_str::<Segment>(&bumped).is_err());
233    }
234
235    #[test]
236    fn segments_path_swaps_the_txt_suffix() {
237        let path = segments_path(Path::new("/m"), "emea", Path::new("a/Q3 Budget.xlsx"));
238        assert_eq!(
239            path,
240            std::path::PathBuf::from("/m/emea/a/Q3 Budget.xlsx.segments.jsonl")
241        );
242    }
243
244    #[test]
245    fn validate_accepts_clean_spans_and_rejects_bad_ones() {
246        let text = "caf\u{e9} au lait";
247        let good = vec![
248            Segment::boundary(SegmentKind::Page, 0),
249            Segment::span(0, text.len(), "document"),
250        ];
251        validate(&good, text).unwrap();
252
253        // An offset inside the two-byte character fails.
254        let bad = vec![Segment::span(0, 4, "document")];
255        assert!(validate(&bad, text).unwrap_err().contains("splits"));
256
257        let past = vec![Segment::span(0, text.len() + 1, "document")];
258        assert!(validate(&past, text).unwrap_err().contains("past"));
259
260        let unordered = vec![Segment::span(3, 5, "row"), Segment::span(0, 2, "row")];
261        assert!(validate(&unordered, text).unwrap_err().contains("order"));
262    }
263}
text_mirror/segments.rs

text_mirror/
segments.rs
