1use serde::{Deserialize, Serialize};
4use unicode_normalization::{UnicodeNormalization, is_nfc_quick};
5
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
7#[serde(rename_all = "camelCase")]
8pub enum BlockKind {
9 Heading,
10 Paragraph,
11 ListItem,
12 TableRow,
13 CodeBlock,
14}
15
16#[derive(Debug, Clone)]
17pub struct Block {
18 pub kind: BlockKind,
19 pub text: String,
21 pub norm: String,
23}
24
25impl Block {
26 pub fn hash(&self) -> [u8; 32] {
28 let mut h = blake3::Hasher::new();
29 h.update(&[self.kind as u8]);
30 h.update(self.norm.as_bytes());
31 *h.finalize().as_bytes()
32 }
33}
34
35fn is_zero_width(c: char) -> bool {
36 matches!(
37 c,
38 '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}' | '\u{00AD}'
39 )
40}
41
42fn nfc(s: String) -> String {
43 if s.is_ascii()
44 || matches!(
45 is_nfc_quick(s.chars()),
46 unicode_normalization::IsNormalized::Yes
47 )
48 {
49 s
50 } else {
51 s.nfc().collect()
52 }
53}
54
55pub fn normalize(s: &str, code: bool) -> String {
57 let cleaned: String = if s.chars().any(is_zero_width) {
58 nfc(s.chars().filter(|&c| !is_zero_width(c)).collect())
59 } else {
60 nfc(s.to_string())
61 };
62 if code {
63 let lines: Vec<&str> = cleaned.lines().map(str::trim_end).collect();
64 lines.join("\n").trim_matches('\n').to_string()
65 } else {
66 let mut out = String::with_capacity(cleaned.len());
67 for w in cleaned.split_whitespace() {
68 if !out.is_empty() {
69 out.push(' ');
70 }
71 out.push_str(w);
72 }
73 out
74 }
75}
76
77fn make(kind: BlockKind, text: String) -> Option<Block> {
78 let norm = normalize(&text, kind == BlockKind::CodeBlock);
79 if norm.is_empty() {
80 return None;
81 }
82 Some(Block {
83 kind,
84 text: text.trim().to_string(),
85 norm,
86 })
87}
88
89fn is_heading(t: &str) -> bool {
90 let hashes = t.bytes().take_while(|&b| b == b'#').count();
91 (1..=6).contains(&hashes) && t[hashes..].chars().next().is_none_or(char::is_whitespace)
92}
93
94fn is_list_item(t: &str) -> bool {
95 let mut it = t.chars();
96 match it.next() {
97 Some('-' | '*' | '+') => it.next().is_some_and(char::is_whitespace),
98 Some(c) if c.is_ascii_digit() => {
99 let rest = t.trim_start_matches(|c: char| c.is_ascii_digit());
100 let mut r = rest.chars();
101 matches!(r.next(), Some('.' | ')')) && r.next().is_some_and(char::is_whitespace)
102 }
103 _ => false,
104 }
105}
106
107fn is_hr(t: &str) -> bool {
108 let mut it = t.chars().filter(|c| !c.is_whitespace());
109 let Some(first) = it.next() else { return false };
110 matches!(first, '-' | '*' | '_' | '=') && {
111 let mut n = 1;
112 it.all(|c| {
113 n += 1;
114 c == first
115 }) && n >= 3
116 }
117}
118
119fn is_table_separator(t: &str) -> bool {
120 t.contains('-') && t.chars().all(|c| matches!(c, '|' | '-' | ':' | ' ' | '\t'))
121}
122
123fn fence_of(t: &str) -> Option<(char, usize)> {
124 let c = t.chars().next()?;
125 if c != '`' && c != '~' {
126 return None;
127 }
128 let n = t.chars().take_while(|&x| x == c).count();
129 (n >= 3).then_some((c, n))
130}
131
132pub fn split_blocks(md: &str) -> Vec<Block> {
133 let mut blocks = Vec::new();
134 let mut cur: Option<(BlockKind, String)> = None;
135 let mut fence: Option<(char, usize, String)> = None;
136
137 fn flush(cur: &mut Option<(BlockKind, String)>, blocks: &mut Vec<Block>) {
138 if let Some((k, t)) = cur.take() {
139 blocks.extend(make(k, t));
140 }
141 }
142 fn append(cur: &mut Option<(BlockKind, String)>, line: &str) {
143 if let Some((_, t)) = cur {
144 t.push('\n');
145 t.push_str(line);
146 }
147 }
148
149 for line in md.lines() {
150 let t = line.trim();
151 if let Some((c, n, buf)) = &mut fence {
152 buf.push('\n');
153 buf.push_str(line);
154 if fence_of(t).is_some_and(|(c2, n2)| c2 == *c && n2 >= *n)
155 && t.chars().all(|x| x == *c)
156 {
157 let text = std::mem::take(buf);
158 fence = None;
159 blocks.extend(make(BlockKind::CodeBlock, text));
160 }
161 continue;
162 }
163 if t.is_empty() {
164 flush(&mut cur, &mut blocks);
165 } else if let Some((c, n)) = fence_of(t) {
166 flush(&mut cur, &mut blocks);
167 fence = Some((c, n, line.to_string()));
168 } else if is_heading(t) {
169 flush(&mut cur, &mut blocks);
170 blocks.extend(make(BlockKind::Heading, t.to_string()));
171 } else if t.starts_with('|') {
172 flush(&mut cur, &mut blocks);
173 if !is_table_separator(t) {
174 blocks.extend(make(BlockKind::TableRow, t.to_string()));
175 }
176 } else if is_hr(t) {
177 flush(&mut cur, &mut blocks);
178 } else if is_list_item(t) {
179 flush(&mut cur, &mut blocks);
180 cur = Some((BlockKind::ListItem, t.to_string()));
181 } else if cur.is_some() {
182 append(&mut cur, t);
183 } else {
184 cur = Some((BlockKind::Paragraph, t.to_string()));
185 }
186 }
187 if let Some((_, _, buf)) = fence {
188 blocks.extend(make(BlockKind::CodeBlock, buf));
189 }
190 flush(&mut cur, &mut blocks);
191 blocks
192}
193
194#[cfg(test)]
195mod tests {
196 use super::*;
197
198 #[test]
199 fn splits_kinds() {
200 let md = "# T\n\npara one\nline two\n\n- a\n- b\n more\n\n| h | v |\n|---|---|\n| x | 1 |\n\n```rs\nlet a = 1;\n\nlet b = 2;\n```\n";
201 let kinds: Vec<_> = split_blocks(md).iter().map(|b| b.kind).collect();
202 use BlockKind::*;
203 assert_eq!(
204 kinds,
205 [
206 Heading, Paragraph, ListItem, ListItem, TableRow, TableRow, CodeBlock
207 ]
208 );
209 }
210
211 #[test]
212 fn normalizes() {
213 let a = split_blocks("caf\u{e9} \u{200B}au\u{a0}lait");
214 let b = split_blocks("cafe\u{301} au lait");
215 assert_eq!(a[0].hash(), b[0].hash());
216 }
217}