sim_codec_doc/
document.rs

1//! Compatibility document chunking over the shared markup IR. Defines
2//! `DocValue`/`DocBlock`/`DocChunk` and their `Expr` projection, parses document
3//! text into blocks (`decode_document`), and splits documents into
4//! provenance-preserving chunks (`chunk`, `ChunkOp`).
5
6use sim_kernel::{Error, Expr, NumberLiteral, Result, Symbol};
7
8use crate::markup::MarkupDoc;
9
10/// A decoded document: its full source text, detected format, and the ordered
11/// [`DocBlock`]s parsed from it.
12///
13/// Block offsets index into [`text`](DocValue::text), so the document carries
14/// provenance for every span it exposes and round-trips back to the original
15/// source.
16#[derive(Clone, Debug, PartialEq, Eq)]
17pub struct DocValue {
18    /// The complete, unmodified source text of the document.
19    pub text: String,
20    /// The detected source format (`DocFormat::Markdown` when any heading was
21    /// seen, otherwise `DocFormat::Text`).
22    pub format: DocFormat,
23    /// The ordered blocks parsed from [`text`](DocValue::text).
24    pub blocks: Vec<DocBlock>,
25}
26
27/// The detected surface format of a decoded document.
28#[derive(Clone, Copy, Debug, PartialEq, Eq)]
29pub enum DocFormat {
30    /// Plain text: no Markdown headings were detected.
31    Text,
32    /// Markdown: at least one `#`-prefixed heading was detected.
33    Markdown,
34}
35
36/// One parsed span of a document, carrying its byte offsets and the heading
37/// path in effect where it appears.
38#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct DocBlock {
40    /// Whether this block is a heading or a paragraph.
41    pub kind: DocBlockKind,
42    /// The block's text: the heading title for headings, the joined paragraph
43    /// lines for paragraphs.
44    pub text: String,
45    /// Inclusive start byte offset of the block within the source text.
46    pub start: usize,
47    /// Exclusive end byte offset of the block within the source text.
48    pub end: usize,
49    /// The chain of enclosing heading titles, outermost first.
50    pub heading_path: Vec<String>,
51    /// The heading depth (1-6) for headings; `None` for paragraphs.
52    pub level: Option<usize>,
53}
54
55/// The category of a [`DocBlock`].
56#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57pub enum DocBlockKind {
58    /// A Markdown heading line.
59    Heading,
60    /// A run of non-blank, non-heading lines.
61    Paragraph,
62}
63
64/// A provenance-preserving slice of a document produced by [`chunk`].
65///
66/// Like [`DocBlock`], the byte offsets index into the originating document's
67/// source text, so a chunk can always be traced back to its origin.
68#[derive(Clone, Debug, PartialEq, Eq)]
69pub struct DocChunk {
70    /// The chunk's text, copied from the source span.
71    pub text: String,
72    /// Inclusive start byte offset of the chunk within the source text.
73    pub start: usize,
74    /// Exclusive end byte offset of the chunk within the source text.
75    pub end: usize,
76    /// The heading path in effect for the source span, outermost first.
77    pub heading_path: Vec<String>,
78}
79
80/// A chunking strategy selecting how [`chunk`] splits a [`DocValue`].
81#[derive(Clone, Copy, Debug, PartialEq, Eq)]
82pub enum ChunkOp {
83    /// Split the whole source into fixed-size windows of at most the given
84    /// number of bytes, respecting character boundaries.
85    Fixed(usize),
86    /// Emit one chunk per paragraph, further splitting any paragraph longer
87    /// than `max` bytes into fixed windows.
88    Recursive {
89        /// The maximum chunk size in bytes before a paragraph is split.
90        max: usize,
91    },
92    /// Emit one chunk per paragraph, each carrying its heading path.
93    Heading,
94}
95
96/// Parse document `text` into a [`DocValue`], recording per-block byte offsets
97/// and heading paths.
98///
99/// Headings are `#`-prefixed lines (levels 1-6); everything else groups into
100/// paragraphs split on blank lines. The format is reported as
101/// `DocFormat::Markdown` when any heading is present, otherwise
102/// `DocFormat::Text`. The returned value's [`text`](DocValue::text) is the
103/// input verbatim.
104///
105/// # Examples
106///
107/// ```
108/// use sim_codec_doc::{DocBlockKind, DocFormat, decode_document};
109///
110/// let doc = decode_document("# Guide\n\nAlpha beta.\n");
111/// assert_eq!(doc.format, DocFormat::Markdown);
112/// assert_eq!(doc.blocks.len(), 2);
113/// assert_eq!(doc.blocks[0].kind, DocBlockKind::Heading);
114/// assert_eq!(doc.blocks[1].text, "Alpha beta.");
115/// assert_eq!(doc.blocks[1].heading_path, vec!["Guide".to_owned()]);
116/// ```
117pub fn decode_document(text: &str) -> DocValue {
118    let mut blocks = Vec::new();
119    let mut headings: Vec<String> = Vec::new();
120    let mut paragraph: Option<(usize, usize, Vec<String>)> = None;
121    let mut saw_heading = false;
122
123    for (line_start, line_end, line) in line_segments(text) {
124        if let Some((level, title)) = heading(line) {
125            flush_paragraph(text, &mut blocks, &mut paragraph);
126            saw_heading = true;
127            while headings.len() >= level {
128                headings.pop();
129            }
130            headings.push(title.clone());
131            blocks.push(DocBlock {
132                kind: DocBlockKind::Heading,
133                text: title,
134                start: line_start,
135                end: line_end,
136                heading_path: headings.clone(),
137                level: Some(level),
138            });
139        } else if line.trim().is_empty() {
140            flush_paragraph(text, &mut blocks, &mut paragraph);
141        } else if let Some((_, end, _)) = &mut paragraph {
142            *end = line_end;
143        } else {
144            paragraph = Some((line_start, line_end, headings.clone()));
145        }
146    }
147
148    flush_paragraph(text, &mut blocks, &mut paragraph);
149
150    DocValue {
151        text: text.to_owned(),
152        format: if saw_heading {
153            DocFormat::Markdown
154        } else {
155            DocFormat::Text
156        },
157        blocks,
158    }
159}
160
161/// Split a decoded `doc` into [`DocChunk`]s according to `op`.
162///
163/// Every chunk preserves its source byte offsets and heading path, so the
164/// resulting chunks round-trip through the general-purpose codecs as ordinary
165/// document slices. See [`ChunkOp`] for the available strategies.
166///
167/// # Examples
168///
169/// ```
170/// use sim_codec_doc::{ChunkOp, chunk, decode_document};
171///
172/// let doc = decode_document("abcdef");
173/// let chunks = chunk(&doc, ChunkOp::Fixed(2));
174/// assert_eq!(chunks.len(), 3);
175/// assert_eq!(chunks[0].text, "ab");
176/// assert_eq!((chunks[0].start, chunks[0].end), (0, 2));
177/// ```
178pub fn chunk(doc: &DocValue, op: ChunkOp) -> Vec<DocChunk> {
179    match op {
180        ChunkOp::Fixed(max) => fixed_range(&doc.text, 0, doc.text.len(), max, Vec::new()),
181        ChunkOp::Recursive { max } => recursive_chunks(doc, max),
182        ChunkOp::Heading => heading_chunks(doc),
183    }
184}
185
186impl DocValue {
187    /// Project this document into its `Expr` map form (`kind: doc`, with
188    /// `format`, `text`, and a list of block maps).
189    pub fn as_expr(&self) -> Expr {
190        Expr::Map(vec![
191            key("kind", Expr::Symbol(Symbol::new("doc"))),
192            key("format", Expr::Symbol(Symbol::new(self.format.name()))),
193            key("text", Expr::String(self.text.clone())),
194            key(
195                "blocks",
196                Expr::List(self.blocks.iter().map(DocBlock::as_expr).collect()),
197            ),
198        ])
199    }
200
201    /// Reconstruct a [`DocValue`] from an `Expr`, accepting a raw string, a
202    /// compatibility `kind: doc` map, or the shared `kind: markup-doc` map. Fails
203    /// closed on any other expression shape.
204    pub fn from_expr(expr: &Expr) -> Result<Self> {
205        match expr {
206            Expr::String(text) => Ok(decode_document(text)),
207            Expr::Map(entries) => {
208                let kind = map_field(entries, "kind")
209                    .ok_or_else(|| Error::Eval("document value requires kind field".to_owned()))?;
210                let Expr::Symbol(symbol) = kind else {
211                    return Err(Error::Eval("document kind must be a symbol".to_owned()));
212                };
213                if symbol.name.as_ref() == "markup-doc" {
214                    let markup = MarkupDoc::from_expr(expr)?;
215                    return Ok(decode_document(&markup.to_source_text()));
216                }
217                if symbol.name.as_ref() != "doc" {
218                    return Err(Error::Eval("document kind must be doc".to_owned()));
219                }
220                let text = map_string(entries, "text")?;
221                Ok(decode_document(text))
222            }
223            _ => Err(Error::TypeMismatch {
224                expected: "document value",
225                found: "non-document",
226            }),
227        }
228    }
229}
230
231impl DocFormat {
232    fn name(self) -> &'static str {
233        match self {
234            Self::Text => "text",
235            Self::Markdown => "markdown",
236        }
237    }
238}
239
240impl DocBlock {
241    fn as_expr(&self) -> Expr {
242        let mut entries = vec![
243            key("kind", Expr::Symbol(Symbol::new(self.kind.name()))),
244            key("text", Expr::String(self.text.clone())),
245            key("start", number(self.start)),
246            key("end", number(self.end)),
247            key("heading_path", string_list(&self.heading_path)),
248        ];
249        if let Some(level) = self.level {
250            entries.push(key("level", number(level)));
251        }
252        Expr::Map(entries)
253    }
254}
255
256impl DocBlockKind {
257    fn name(self) -> &'static str {
258        match self {
259            Self::Heading => "heading",
260            Self::Paragraph => "paragraph",
261        }
262    }
263}
264
265impl DocChunk {
266    /// Project this chunk into its `Expr` map form (`kind: doc-chunk`, with
267    /// `text`, `start`, `end`, and `heading_path`).
268    pub fn as_expr(&self) -> Expr {
269        Expr::Map(vec![
270            key("kind", Expr::Symbol(Symbol::new("doc-chunk"))),
271            key("text", Expr::String(self.text.clone())),
272            key("start", number(self.start)),
273            key("end", number(self.end)),
274            key("heading_path", string_list(&self.heading_path)),
275        ])
276    }
277}
278
279fn recursive_chunks(doc: &DocValue, max: usize) -> Vec<DocChunk> {
280    if max == 0 {
281        return Vec::new();
282    }
283    let mut chunks = Vec::new();
284    for block in doc
285        .blocks
286        .iter()
287        .filter(|block| block.kind == DocBlockKind::Paragraph)
288    {
289        if block.end.saturating_sub(block.start) <= max {
290            chunks.push(chunk_for_range(
291                &doc.text,
292                block.start,
293                block.end,
294                block.heading_path.clone(),
295            ));
296        } else {
297            chunks.extend(fixed_range(
298                &doc.text,
299                block.start,
300                block.end,
301                max,
302                block.heading_path.clone(),
303            ));
304        }
305    }
306    if chunks.is_empty() && !doc.text.is_empty() {
307        chunks.extend(fixed_range(&doc.text, 0, doc.text.len(), max, Vec::new()));
308    }
309    chunks
310}
311
312fn heading_chunks(doc: &DocValue) -> Vec<DocChunk> {
313    let chunks = doc
314        .blocks
315        .iter()
316        .filter(|block| block.kind == DocBlockKind::Paragraph)
317        .map(|block| {
318            chunk_for_range(
319                &doc.text,
320                block.start,
321                block.end,
322                block.heading_path.clone(),
323            )
324        })
325        .collect::<Vec<_>>();
326    if chunks.is_empty() && !doc.text.is_empty() {
327        vec![chunk_for_range(&doc.text, 0, doc.text.len(), Vec::new())]
328    } else {
329        chunks
330    }
331}
332
333fn fixed_range(
334    text: &str,
335    start: usize,
336    end: usize,
337    max: usize,
338    heading_path: Vec<String>,
339) -> Vec<DocChunk> {
340    if max == 0 || start >= end {
341        return Vec::new();
342    }
343    let mut chunks = Vec::new();
344    let mut cursor = start;
345    while cursor < end {
346        let next = char_boundary_at_or_before(text, cursor.saturating_add(max).min(end), cursor);
347        let next = if next == cursor {
348            next_char_boundary_after(text, cursor).min(end)
349        } else {
350            next
351        };
352        chunks.push(chunk_for_range(text, cursor, next, heading_path.clone()));
353        cursor = next;
354    }
355    chunks
356}
357
358fn chunk_for_range(text: &str, start: usize, end: usize, heading_path: Vec<String>) -> DocChunk {
359    DocChunk {
360        text: text[start..end].to_owned(),
361        start,
362        end,
363        heading_path,
364    }
365}
366
367fn char_boundary_at_or_before(text: &str, mut index: usize, floor: usize) -> usize {
368    while index > floor && !text.is_char_boundary(index) {
369        index -= 1;
370    }
371    index
372}
373
374fn next_char_boundary_after(text: &str, index: usize) -> usize {
375    text[index..]
376        .char_indices()
377        .nth(1)
378        .map(|(offset, _)| index + offset)
379        .unwrap_or(text.len())
380}
381
382fn flush_paragraph(
383    source: &str,
384    blocks: &mut Vec<DocBlock>,
385    paragraph: &mut Option<(usize, usize, Vec<String>)>,
386) {
387    let Some((start, end, heading_path)) = paragraph.take() else {
388        return;
389    };
390    blocks.push(DocBlock {
391        kind: DocBlockKind::Paragraph,
392        text: source[start..end].to_owned(),
393        start,
394        end,
395        heading_path,
396        level: None,
397    });
398}
399
400fn line_segments(source: &str) -> Vec<(usize, usize, &str)> {
401    let mut segments = Vec::new();
402    let mut offset = 0;
403    for segment in source.split_inclusive('\n') {
404        let start = offset;
405        offset += segment.len();
406        let mut end = offset;
407        if segment.ends_with('\n') {
408            end -= 1;
409            if end > start && source.as_bytes()[end - 1] == b'\r' {
410                end -= 1;
411            }
412        }
413        segments.push((start, end, &source[start..end]));
414    }
415    segments
416}
417
418fn heading(line: &str) -> Option<(usize, String)> {
419    let trimmed = line.trim_start();
420    let level = trimmed.bytes().take_while(|byte| *byte == b'#').count();
421    if level == 0 || level > 6 {
422        return None;
423    }
424    let rest = &trimmed[level..];
425    if !rest.starts_with(char::is_whitespace) {
426        return None;
427    }
428    let title = rest.trim();
429    (!title.is_empty()).then(|| (level, title.to_owned()))
430}
431
432fn key(name: &str, value: Expr) -> (Expr, Expr) {
433    (Expr::Symbol(Symbol::new(name)), value)
434}
435
436fn number(value: usize) -> Expr {
437    Expr::Number(NumberLiteral {
438        domain: Symbol::qualified("numbers", "f64"),
439        canonical: value.to_string(),
440    })
441}
442
443fn string_list(values: &[String]) -> Expr {
444    Expr::List(values.iter().cloned().map(Expr::String).collect())
445}
446
447use sim_value::access::entry_field as map_field;
448
449fn map_string<'a>(entries: &'a [(Expr, Expr)], key: &str) -> Result<&'a str> {
450    match map_field(entries, key) {
451        Some(Expr::String(value)) => Ok(value),
452        Some(_) => Err(Error::Eval(format!(
453            "document {key} field must be a string"
454        ))),
455        None => Err(Error::Eval(format!("document value requires {key} field"))),
456    }
457}
sim_codec_doc/document.rs

sim_codec_doc/
document.rs
