sim_codec_doc/
document.rs

1//! The document model and chunking. Defines `DocValue`/`DocBlock`/`DocChunk`
2//! and their `Expr` projection, parses document text into blocks
3//! (`decode_document`), and splits documents into provenance-preserving chunks
4//! (`chunk`, `ChunkOp`).
5
6use sim_kernel::{Error, Expr, NumberLiteral, Result, Symbol};
7
8/// A decoded document: its full source text, detected format, and the ordered
9/// [`DocBlock`]s parsed from it.
10///
11/// Block offsets index into [`text`](DocValue::text), so the document carries
12/// provenance for every span it exposes and round-trips back to the original
13/// source.
14#[derive(Clone, Debug, PartialEq, Eq)]
15pub struct DocValue {
16    /// The complete, unmodified source text of the document.
17    pub text: String,
18    /// The detected source format (`DocFormat::Markdown` when any heading was
19    /// seen, otherwise `DocFormat::Text`).
20    pub format: DocFormat,
21    /// The ordered blocks parsed from [`text`](DocValue::text).
22    pub blocks: Vec<DocBlock>,
23}
24
25/// The detected surface format of a decoded document.
26#[derive(Clone, Copy, Debug, PartialEq, Eq)]
27pub enum DocFormat {
28    /// Plain text: no Markdown headings were detected.
29    Text,
30    /// Markdown: at least one `#`-prefixed heading was detected.
31    Markdown,
32}
33
34/// One parsed span of a document, carrying its byte offsets and the heading
35/// path in effect where it appears.
36#[derive(Clone, Debug, PartialEq, Eq)]
37pub struct DocBlock {
38    /// Whether this block is a heading or a paragraph.
39    pub kind: DocBlockKind,
40    /// The block's text: the heading title for headings, the joined paragraph
41    /// lines for paragraphs.
42    pub text: String,
43    /// Inclusive start byte offset of the block within the source text.
44    pub start: usize,
45    /// Exclusive end byte offset of the block within the source text.
46    pub end: usize,
47    /// The chain of enclosing heading titles, outermost first.
48    pub heading_path: Vec<String>,
49    /// The heading depth (1-6) for headings; `None` for paragraphs.
50    pub level: Option<usize>,
51}
52
53/// The category of a [`DocBlock`].
54#[derive(Clone, Copy, Debug, PartialEq, Eq)]
55pub enum DocBlockKind {
56    /// A Markdown heading line.
57    Heading,
58    /// A run of non-blank, non-heading lines.
59    Paragraph,
60}
61
62/// A provenance-preserving slice of a document produced by [`chunk`].
63///
64/// Like [`DocBlock`], the byte offsets index into the originating document's
65/// source text, so a chunk can always be traced back to its origin.
66#[derive(Clone, Debug, PartialEq, Eq)]
67pub struct DocChunk {
68    /// The chunk's text, copied from the source span.
69    pub text: String,
70    /// Inclusive start byte offset of the chunk within the source text.
71    pub start: usize,
72    /// Exclusive end byte offset of the chunk within the source text.
73    pub end: usize,
74    /// The heading path in effect for the source span, outermost first.
75    pub heading_path: Vec<String>,
76}
77
78/// A chunking strategy selecting how [`chunk`] splits a [`DocValue`].
79#[derive(Clone, Copy, Debug, PartialEq, Eq)]
80pub enum ChunkOp {
81    /// Split the whole source into fixed-size windows of at most the given
82    /// number of bytes, respecting character boundaries.
83    Fixed(usize),
84    /// Emit one chunk per paragraph, further splitting any paragraph longer
85    /// than `max` bytes into fixed windows.
86    Recursive {
87        /// The maximum chunk size in bytes before a paragraph is split.
88        max: usize,
89    },
90    /// Emit one chunk per paragraph, each carrying its heading path.
91    Heading,
92}
93
94/// Parse document `text` into a [`DocValue`], recording per-block byte offsets
95/// and heading paths.
96///
97/// Headings are `#`-prefixed lines (levels 1-6); everything else groups into
98/// paragraphs split on blank lines. The format is reported as
99/// `DocFormat::Markdown` when any heading is present, otherwise
100/// `DocFormat::Text`. The returned value's [`text`](DocValue::text) is the
101/// input verbatim.
102///
103/// # Examples
104///
105/// ```
106/// use sim_codec_doc::{DocBlockKind, DocFormat, decode_document};
107///
108/// let doc = decode_document("# Guide\n\nAlpha beta.\n");
109/// assert_eq!(doc.format, DocFormat::Markdown);
110/// assert_eq!(doc.blocks.len(), 2);
111/// assert_eq!(doc.blocks[0].kind, DocBlockKind::Heading);
112/// assert_eq!(doc.blocks[1].text, "Alpha beta.");
113/// assert_eq!(doc.blocks[1].heading_path, vec!["Guide".to_owned()]);
114/// ```
115pub fn decode_document(text: &str) -> DocValue {
116    let mut blocks = Vec::new();
117    let mut headings: Vec<String> = Vec::new();
118    let mut paragraph: Option<(usize, usize, Vec<String>)> = None;
119    let mut saw_heading = false;
120
121    for (line_start, line_end, line) in line_segments(text) {
122        if let Some((level, title)) = heading(line) {
123            flush_paragraph(text, &mut blocks, &mut paragraph);
124            saw_heading = true;
125            while headings.len() >= level {
126                headings.pop();
127            }
128            headings.push(title.clone());
129            blocks.push(DocBlock {
130                kind: DocBlockKind::Heading,
131                text: title,
132                start: line_start,
133                end: line_end,
134                heading_path: headings.clone(),
135                level: Some(level),
136            });
137        } else if line.trim().is_empty() {
138            flush_paragraph(text, &mut blocks, &mut paragraph);
139        } else if let Some((_, end, _)) = &mut paragraph {
140            *end = line_end;
141        } else {
142            paragraph = Some((line_start, line_end, headings.clone()));
143        }
144    }
145
146    flush_paragraph(text, &mut blocks, &mut paragraph);
147
148    DocValue {
149        text: text.to_owned(),
150        format: if saw_heading {
151            DocFormat::Markdown
152        } else {
153            DocFormat::Text
154        },
155        blocks,
156    }
157}
158
159/// Split a decoded `doc` into [`DocChunk`]s according to `op`.
160///
161/// Every chunk preserves its source byte offsets and heading path, so the
162/// resulting chunks round-trip through the general-purpose codecs as ordinary
163/// document slices. See [`ChunkOp`] for the available strategies.
164///
165/// # Examples
166///
167/// ```
168/// use sim_codec_doc::{ChunkOp, chunk, decode_document};
169///
170/// let doc = decode_document("abcdef");
171/// let chunks = chunk(&doc, ChunkOp::Fixed(2));
172/// assert_eq!(chunks.len(), 3);
173/// assert_eq!(chunks[0].text, "ab");
174/// assert_eq!((chunks[0].start, chunks[0].end), (0, 2));
175/// ```
176pub fn chunk(doc: &DocValue, op: ChunkOp) -> Vec<DocChunk> {
177    match op {
178        ChunkOp::Fixed(max) => fixed_range(&doc.text, 0, doc.text.len(), max, Vec::new()),
179        ChunkOp::Recursive { max } => recursive_chunks(doc, max),
180        ChunkOp::Heading => heading_chunks(doc),
181    }
182}
183
184impl DocValue {
185    /// Project this document into its `Expr` map form (`kind: doc`, with
186    /// `format`, `text`, and a list of block maps).
187    pub fn as_expr(&self) -> Expr {
188        Expr::Map(vec![
189            key("kind", Expr::Symbol(Symbol::new("doc"))),
190            key("format", Expr::Symbol(Symbol::new(self.format.name()))),
191            key("text", Expr::String(self.text.clone())),
192            key(
193                "blocks",
194                Expr::List(self.blocks.iter().map(DocBlock::as_expr).collect()),
195            ),
196        ])
197    }
198
199    /// Reconstruct a [`DocValue`] from an `Expr`, accepting either a raw string
200    /// (decoded as document text) or a `kind: doc` map (decoded from its `text`
201    /// field). Fails closed on any other expression shape.
202    pub fn from_expr(expr: &Expr) -> Result<Self> {
203        match expr {
204            Expr::String(text) => Ok(decode_document(text)),
205            Expr::Map(entries) => {
206                let kind = map_field(entries, "kind")
207                    .ok_or_else(|| Error::Eval("document value requires kind field".to_owned()))?;
208                let Expr::Symbol(symbol) = kind else {
209                    return Err(Error::Eval("document kind must be a symbol".to_owned()));
210                };
211                if symbol.name.as_ref() != "doc" {
212                    return Err(Error::Eval("document kind must be doc".to_owned()));
213                }
214                let text = map_string(entries, "text")?;
215                Ok(decode_document(text))
216            }
217            _ => Err(Error::TypeMismatch {
218                expected: "document value",
219                found: "non-document",
220            }),
221        }
222    }
223}
224
225impl DocFormat {
226    fn name(self) -> &'static str {
227        match self {
228            Self::Text => "text",
229            Self::Markdown => "markdown",
230        }
231    }
232}
233
234impl DocBlock {
235    fn as_expr(&self) -> Expr {
236        let mut entries = vec![
237            key("kind", Expr::Symbol(Symbol::new(self.kind.name()))),
238            key("text", Expr::String(self.text.clone())),
239            key("start", number(self.start)),
240            key("end", number(self.end)),
241            key("heading_path", string_list(&self.heading_path)),
242        ];
243        if let Some(level) = self.level {
244            entries.push(key("level", number(level)));
245        }
246        Expr::Map(entries)
247    }
248}
249
250impl DocBlockKind {
251    fn name(self) -> &'static str {
252        match self {
253            Self::Heading => "heading",
254            Self::Paragraph => "paragraph",
255        }
256    }
257}
258
259impl DocChunk {
260    /// Project this chunk into its `Expr` map form (`kind: doc-chunk`, with
261    /// `text`, `start`, `end`, and `heading_path`).
262    pub fn as_expr(&self) -> Expr {
263        Expr::Map(vec![
264            key("kind", Expr::Symbol(Symbol::new("doc-chunk"))),
265            key("text", Expr::String(self.text.clone())),
266            key("start", number(self.start)),
267            key("end", number(self.end)),
268            key("heading_path", string_list(&self.heading_path)),
269        ])
270    }
271}
272
273fn recursive_chunks(doc: &DocValue, max: usize) -> Vec<DocChunk> {
274    if max == 0 {
275        return Vec::new();
276    }
277    let mut chunks = Vec::new();
278    for block in doc
279        .blocks
280        .iter()
281        .filter(|block| block.kind == DocBlockKind::Paragraph)
282    {
283        if block.end.saturating_sub(block.start) <= max {
284            chunks.push(chunk_for_range(
285                &doc.text,
286                block.start,
287                block.end,
288                block.heading_path.clone(),
289            ));
290        } else {
291            chunks.extend(fixed_range(
292                &doc.text,
293                block.start,
294                block.end,
295                max,
296                block.heading_path.clone(),
297            ));
298        }
299    }
300    if chunks.is_empty() && !doc.text.is_empty() {
301        chunks.extend(fixed_range(&doc.text, 0, doc.text.len(), max, Vec::new()));
302    }
303    chunks
304}
305
306fn heading_chunks(doc: &DocValue) -> Vec<DocChunk> {
307    let chunks = doc
308        .blocks
309        .iter()
310        .filter(|block| block.kind == DocBlockKind::Paragraph)
311        .map(|block| {
312            chunk_for_range(
313                &doc.text,
314                block.start,
315                block.end,
316                block.heading_path.clone(),
317            )
318        })
319        .collect::<Vec<_>>();
320    if chunks.is_empty() && !doc.text.is_empty() {
321        vec![chunk_for_range(&doc.text, 0, doc.text.len(), Vec::new())]
322    } else {
323        chunks
324    }
325}
326
327fn fixed_range(
328    text: &str,
329    start: usize,
330    end: usize,
331    max: usize,
332    heading_path: Vec<String>,
333) -> Vec<DocChunk> {
334    if max == 0 || start >= end {
335        return Vec::new();
336    }
337    let mut chunks = Vec::new();
338    let mut cursor = start;
339    while cursor < end {
340        let next = char_boundary_at_or_before(text, cursor.saturating_add(max).min(end), cursor);
341        let next = if next == cursor {
342            next_char_boundary_after(text, cursor).min(end)
343        } else {
344            next
345        };
346        chunks.push(chunk_for_range(text, cursor, next, heading_path.clone()));
347        cursor = next;
348    }
349    chunks
350}
351
352fn chunk_for_range(text: &str, start: usize, end: usize, heading_path: Vec<String>) -> DocChunk {
353    DocChunk {
354        text: text[start..end].to_owned(),
355        start,
356        end,
357        heading_path,
358    }
359}
360
361fn char_boundary_at_or_before(text: &str, mut index: usize, floor: usize) -> usize {
362    while index > floor && !text.is_char_boundary(index) {
363        index -= 1;
364    }
365    index
366}
367
368fn next_char_boundary_after(text: &str, index: usize) -> usize {
369    text[index..]
370        .char_indices()
371        .nth(1)
372        .map(|(offset, _)| index + offset)
373        .unwrap_or(text.len())
374}
375
376fn flush_paragraph(
377    source: &str,
378    blocks: &mut Vec<DocBlock>,
379    paragraph: &mut Option<(usize, usize, Vec<String>)>,
380) {
381    let Some((start, end, heading_path)) = paragraph.take() else {
382        return;
383    };
384    blocks.push(DocBlock {
385        kind: DocBlockKind::Paragraph,
386        text: source[start..end].to_owned(),
387        start,
388        end,
389        heading_path,
390        level: None,
391    });
392}
393
394fn line_segments(source: &str) -> Vec<(usize, usize, &str)> {
395    let mut segments = Vec::new();
396    let mut offset = 0;
397    for segment in source.split_inclusive('\n') {
398        let start = offset;
399        offset += segment.len();
400        let mut end = offset;
401        if segment.ends_with('\n') {
402            end -= 1;
403            if end > start && source.as_bytes()[end - 1] == b'\r' {
404                end -= 1;
405            }
406        }
407        segments.push((start, end, &source[start..end]));
408    }
409    segments
410}
411
412fn heading(line: &str) -> Option<(usize, String)> {
413    let trimmed = line.trim_start();
414    let level = trimmed.bytes().take_while(|byte| *byte == b'#').count();
415    if level == 0 || level > 6 {
416        return None;
417    }
418    let rest = &trimmed[level..];
419    if !rest.starts_with(char::is_whitespace) {
420        return None;
421    }
422    let title = rest.trim();
423    (!title.is_empty()).then(|| (level, title.to_owned()))
424}
425
426fn key(name: &str, value: Expr) -> (Expr, Expr) {
427    (Expr::Symbol(Symbol::new(name)), value)
428}
429
430fn number(value: usize) -> Expr {
431    Expr::Number(NumberLiteral {
432        domain: Symbol::qualified("numbers", "f64"),
433        canonical: value.to_string(),
434    })
435}
436
437fn string_list(values: &[String]) -> Expr {
438    Expr::List(values.iter().cloned().map(Expr::String).collect())
439}
440
441use sim_value::access::entry_field as map_field;
442
443fn map_string<'a>(entries: &'a [(Expr, Expr)], key: &str) -> Result<&'a str> {
444    match map_field(entries, key) {
445        Some(Expr::String(value)) => Ok(value),
446        Some(_) => Err(Error::Eval(format!(
447            "document {key} field must be a string"
448        ))),
449        None => Err(Error::Eval(format!("document value requires {key} field"))),
450    }
451}
sim_codec_doc/document.rs

sim_codec_doc/
document.rs
