grepdown_lib/db/
parse.rs

1use crate::error::Result;
2use ignore::WalkBuilder;
3use pulldown_cmark::{Event, Parser, Tag};
4use rayon::prelude::*;
5use rusqlite::params;
6use std::{
7    collections::{HashMap, HashSet},
8    fs,
9    os::unix::fs::MetadataExt,
10    path::Path,
11};
12
13use crate::frontmatter::{extract_tags, parse_frontmatter};
14use crate::project::MDDBProject;
15
16const STMT_MTIME: &str = "SELECT path, mtime, content_hash FROM documents";
17const STMT_DEL_FTS: &str = "DELETE FROM documents_fts WHERE path = ?1";
18const STMT_INS_FTS: &str = "INSERT INTO documents_fts (path, body) VALUES (?1, ?2)";
19const STMT_UPD_META: &str = "INSERT INTO documents (path, mtime, content_hash) VALUES (?1, ?2, ?3) ON CONFLICT(path) DO UPDATE SET mtime = excluded.mtime, content_hash = excluded.content_hash, version = CASE WHEN content_hash != excluded.content_hash THEN version + 1 ELSE version END";
20const STMT_DEL_TAGS: &str = "DELETE FROM tags_fts WHERE path = ?1";
21const STMT_INS_TAGS: &str = "INSERT INTO tags_fts (path, tags) VALUES (?1, ?2)";
22const STMT_DEL_LINKS: &str = "DELETE FROM links WHERE from_id = ?1";
23const STMT_INS_LINK: &str = "INSERT INTO links (from_id, to_id, raw_target, pinned_version, anchor) VALUES (?1, ?2, ?3, (SELECT version FROM documents WHERE path = ?2), ?4)";
24const STMT_DEL_CITATIONS: &str = "DELETE FROM citations WHERE from_id = ?1";
25const STMT_INS_CITATION: &str =
26    "INSERT INTO citations (from_id, url, raw_target) VALUES (?1, ?2, ?3)";
27const STMT_DEL_BROKEN: &str = "DELETE FROM broken_links WHERE from_id = ?1";
28const STMT_INS_BROKEN: &str = "INSERT INTO broken_links (from_id, raw_target) VALUES (?1, ?2)";
29const STMT_DEL_HEADINGS: &str = "DELETE FROM headings WHERE path = ?1";
30const STMT_INS_HEADING: &str =
31    "INSERT INTO headings (path, level, text, anchor) VALUES (?1, ?2, ?3, ?4)";
32const STMT_DEL_HEADINGS_FTS: &str = "DELETE FROM headings_fts WHERE path = ?1";
33const STMT_INS_HEADINGS_FTS: &str = "INSERT INTO headings_fts (path, headings) VALUES (?1, ?2)";
34
35/// Extract all links from markdown content.
36/// Returns (target, is_external, anchor) where is_external=true means citation (URL).
37fn extract_links(content: &str) -> Vec<(String, bool, Option<String>)> {
38    Parser::new(content)
39        .filter_map(|event| match event {
40            Event::Start(Tag::Link { dest_url, .. })
41            | Event::Start(Tag::Image { dest_url, .. }) => {
42                let url = dest_url.to_string();
43                let is_external =
44                    url.contains("://") || url.starts_with("mailto:") || url.starts_with("//");
45
46                // Extract anchor (fragment) from URL
47                let (target, anchor) = if let Some(hash_pos) = url.find('#') {
48                    let anchor = url[hash_pos + 1..].to_string();
49                    let target = url[..hash_pos].to_string();
50                    (target, Some(anchor))
51                } else {
52                    (url, None)
53                };
54
55                Some((target, is_external, anchor))
56            }
57            _ => None,
58        })
59        .collect()
60}
61
62/// Extract all headings from markdown content.
63/// Returns (level, text) pairs where level is 1-6.
64fn extract_headings(content: &str) -> Vec<(i32, String)> {
65    let mut headings = Vec::new();
66    let mut in_heading = false;
67    let mut current_level = 0;
68    let mut current_text = String::new();
69
70    for event in Parser::new(content) {
71        match event {
72            Event::Start(Tag::Heading { level, .. }) => {
73                in_heading = true;
74                current_level = level as i32;
75                current_text.clear();
76            }
77            Event::Text(text) if in_heading => {
78                current_text.push_str(&text);
79            }
80            Event::End(_) if in_heading => {
81                headings.push((current_level, current_text.clone()));
82                in_heading = false;
83            }
84            _ => {}
85        }
86    }
87
88    headings
89}
90
91/// Convert heading text to a URL-friendly anchor slug.
92/// - Lowercase
93/// - Replace spaces with hyphens
94/// - Remove non-alphanumeric characters (except hyphens)
95/// - Collapse consecutive hyphens
96fn slugify(text: &str) -> String {
97    let mut result = String::with_capacity(text.len());
98    let mut last_was_hyphen = false;
99
100    for c in text.chars() {
101        if c.is_alphanumeric() {
102            result.push(c.to_ascii_lowercase());
103            last_was_hyphen = false;
104        } else if c == ' ' || c == '-' {
105            if !last_was_hyphen && !result.is_empty() {
106                result.push('-');
107                last_was_hyphen = true;
108            }
109        }
110    }
111
112    // Remove trailing hyphen
113    if result.ends_with('-') {
114        result.pop();
115    }
116
117    result
118}
119
120/// Result of indexing a single document in the parallel phase.
121/// Holds the fully-resolved link data so the write transaction never touches the filesystem.
122struct IndexedDoc {
123    path: String,
124    mtime: i64,
125    content: String,
126    hash: Vec<u8>,
127    tags: String,
128    /// (resolved_target, raw_target, anchor) — deduped by resolved_target
129    resolved_links: Vec<(String, String, Option<String>)>,
130    /// External URLs — deduped
131    citations: Vec<String>,
132    /// Raw targets that couldn't be resolved
133    broken_raw: Vec<String>,
134    /// (level, text, anchor)
135    headings: Vec<(i32, String, String)>,
136}
137
138/// Resolve a bundle-relative link target to a canonical document path using only
139/// in-memory path-set membership — no filesystem access.
140/// Returns `None` if the target isn't in `current_paths` (i.e., the link is broken
141/// or escapes the project root).
142fn resolve_in_set(
143    current_paths: &HashSet<String>,
144    base_path: &str,
145    target: &str,
146) -> Option<String> {
147    let base_dir = Path::new(base_path).parent()?;
148    let normalized = normalize_path(&base_dir.join(target));
149
150    let direct = normalized.with_extension("md");
151    let direct_s = direct.to_string_lossy().into_owned();
152    if current_paths.contains(&direct_s) {
153        return Some(direct_s);
154    }
155
156    let index = normalized.join("index.md");
157    let index_s = index.to_string_lossy().into_owned();
158    if current_paths.contains(&index_s) {
159        return Some(index_s);
160    }
161
162    None
163}
164
165/// Normalize a path by resolving . and .. components without following symlinks.
166fn normalize_path(path: &Path) -> std::path::PathBuf {
167    let mut normalized = std::path::PathBuf::new();
168    for component in path.components() {
169        match component {
170            std::path::Component::ParentDir => {
171                normalized.pop();
172            }
173            std::path::Component::CurDir => {
174                // Skip .
175            }
176            _ => {
177                normalized.push(component);
178            }
179        }
180    }
181    normalized
182}
183
184/// Result of processing a file whose mtime changed.
185/// `Changed` = content differs → full re-index.
186/// `Unchanged` = content matches → only mtime needs write-back.
187enum ParseResult {
188    Changed(IndexedDoc),
189    Unchanged {
190        path: String,
191        mtime: i64,
192        hash: Vec<u8>,
193    },
194}
195
196impl MDDBProject {
197    /// Refresh the database and index files not seen before
198    pub fn refresh(&self) -> Result<Vec<(String, i64)>> {
199        let root = self.get_root();
200        let mut known = HashMap::<String, (i64, Vec<u8>)>::new();
201        let conn = self.get_conn();
202
203        // Load known mtimes into memory
204        {
205            let mut stmt = conn.prepare(STMT_MTIME)?;
206            let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
207
208            for r in rows {
209                let (path, mtime, content_hash): (String, i64, Vec<u8>) = r?;
210                known.insert(path, (mtime, content_hash));
211            }
212        }
213
214        // Walk and diff — respects .gitignore, .ignore, .rgignore automatically
215        let mut changed: Vec<(String, i64)> = Vec::new();
216        let mut current_paths = HashSet::new();
217        let mut walked = 0usize;
218
219        for entry in WalkBuilder::new(self.get_root())
220            .build()
221            .filter_map(|e| e.ok())
222            .filter(|e| e.path().extension().map_or(false, |x| x == "md"))
223        {
224            let meta = match entry.metadata() {
225                Ok(m) => m,
226                Err(_) => continue,
227            };
228            let mtime = meta.mtime();
229            let abs_path = entry.path();
230            let rel_path = abs_path.strip_prefix(root).unwrap_or(abs_path);
231            let path = rel_path.to_string_lossy().into_owned();
232
233            current_paths.insert(path.clone());
234            walked += 1;
235            match known.get(&path) {
236                Some(&(old_mtime, _)) if old_mtime == mtime => {}
237                _ => changed.push((path, mtime)),
238            }
239        }
240        log::debug!("Walked {} files, {} changed", walked, changed.len());
241
242        // Detect deleted files (known from DB but no longer on disk)
243        let deleted: Vec<String> = known
244            .keys()
245            .filter(|k| !current_paths.contains(k.as_str()))
246            .cloned()
247            .collect();
248
249        // Parallel read changed files (level-2: skip link resolution if content unchanged)
250        let results: Vec<ParseResult> = changed
251            .par_iter()
252            .map(|(path, mtime)| {
253                let content = fs::read_to_string(Path::new(root).join(path)).unwrap_or_else(|e| {
254                    log::warn!("Failed to read {}: {}", path, e);
255                    String::new()
256                });
257                let hash = blake3::hash(content.as_bytes()).as_bytes().to_vec();
258
259                if let Some((_, old_hash)) = known.get(path) {
260                    if *old_hash == hash {
261                        // Content unchanged — record mtime-only touch, skip parsing
262                        return ParseResult::Unchanged {
263                            path: path.clone(),
264                            mtime: *mtime,
265                            hash,
266                        };
267                    }
268                }
269
270                let tags = parse_frontmatter(&content)
271                    .map(|fm| extract_tags(&fm))
272                    .unwrap_or_default();
273                let tags_str = tags.join(" ");
274
275                let mut links = extract_links(&content);
276                links.sort();
277                links.dedup();
278
279                // Resolve all links via in-memory set membership — zero syscalls
280                let mut resolved_map: HashMap<String, (String, Option<String>)> = HashMap::new();
281                let mut citation_set: HashSet<String> = HashSet::new();
282                let mut broken_raw: Vec<String> = Vec::new();
283
284                for (target, is_external, anchor) in &links {
285                    if *is_external {
286                        citation_set.insert(target.clone());
287                    } else {
288                        match resolve_in_set(&current_paths, path, target) {
289                            Some(resolved) => {
290                                resolved_map.insert(resolved, (target.clone(), anchor.clone()));
291                            }
292                            None => {
293                                broken_raw.push(target.clone());
294                            }
295                        }
296                    }
297                }
298
299                // Extract headings and generate anchors
300                let raw_headings = extract_headings(&content);
301                let headings: Vec<(i32, String, String)> = raw_headings
302                    .into_iter()
303                    .map(|(level, text)| {
304                        let anchor = slugify(&text);
305                        (level, text, anchor)
306                    })
307                    .collect();
308
309                ParseResult::Changed(IndexedDoc {
310                    path: path.clone(),
311                    mtime: *mtime,
312                    content,
313                    hash,
314                    tags: tags_str,
315                    resolved_links: resolved_map
316                        .into_iter()
317                        .map(|(resolved, (raw, anchor))| (resolved, raw, anchor))
318                        .collect(),
319                    citations: citation_set.into_iter().collect(),
320                    broken_raw,
321                    headings,
322                })
323            })
324            .collect();
325
326        // Split into changed-docs (full re-index) and touch-only (mtime write-back)
327        let mut changed_docs: Vec<IndexedDoc> = Vec::new();
328        let mut touch: Vec<(String, i64, Vec<u8>)> = Vec::new();
329        for r in results {
330            match r {
331                ParseResult::Changed(doc) => changed_docs.push(doc),
332                ParseResult::Unchanged { path, mtime, hash } => touch.push((path, mtime, hash)),
333            }
334        }
335
336        // Rebuild changed return value from actually-reindexed docs
337        changed = changed_docs
338            .iter()
339            .map(|doc| (doc.path.clone(), doc.mtime))
340            .collect();
341        log::info!(
342            "Indexed {} files, {} mtime-only touches",
343            changed_docs.len(),
344            touch.len()
345        );
346
347        // Do a single transaction for the whole batch
348        let tx = conn.unchecked_transaction()?;
349        {
350            let mut del_fts = tx.prepare(STMT_DEL_FTS)?;
351            let mut ins_fts = tx.prepare(STMT_INS_FTS)?;
352            let mut upsert_meta = tx.prepare(STMT_UPD_META)?;
353            let mut del_tags = tx.prepare(STMT_DEL_TAGS)?;
354            let mut ins_tags = tx.prepare(STMT_INS_TAGS)?;
355
356            // Phase 1: Upsert all documents first (so FK constraints pass for links)
357            for doc in &changed_docs {
358                del_fts.execute(params![doc.path])?;
359                ins_fts.execute(params![doc.path, doc.content])?;
360                upsert_meta.execute(params![doc.path, doc.mtime, doc.hash])?;
361                del_tags.execute(params![doc.path])?;
362                if !doc.tags.is_empty() {
363                    ins_tags.execute(params![doc.path, doc.tags])?;
364                }
365            }
366
367            // Phase 2: Insert links/citations/broken (all documents exist, resolution done in parallel phase)
368            let mut del_links = tx.prepare(STMT_DEL_LINKS)?;
369            let mut ins_link = tx.prepare(STMT_INS_LINK)?;
370            let mut del_citations = tx.prepare(STMT_DEL_CITATIONS)?;
371            let mut ins_citation = tx.prepare(STMT_INS_CITATION)?;
372            let mut del_broken = tx.prepare(STMT_DEL_BROKEN)?;
373            let mut ins_broken = tx.prepare(STMT_INS_BROKEN)?;
374            for doc in &changed_docs {
375                del_links.execute(params![doc.path])?;
376                del_citations.execute(params![doc.path])?;
377                del_broken.execute(params![doc.path])?;
378
379                for (resolved, raw, anchor) in &doc.resolved_links {
380                    ins_link.execute(params![doc.path, resolved, raw, anchor])?;
381                }
382                for url in &doc.citations {
383                    ins_citation.execute(params![doc.path, url, url])?;
384                }
385                for raw in &doc.broken_raw {
386                    ins_broken.execute(params![doc.path, raw])?;
387                }
388            }
389
390            // Phase 3: Insert headings (structured + FTS)
391            let mut del_headings = tx.prepare(STMT_DEL_HEADINGS)?;
392            let mut ins_heading = tx.prepare(STMT_INS_HEADING)?;
393            let mut del_headings_fts = tx.prepare(STMT_DEL_HEADINGS_FTS)?;
394            let mut ins_headings_fts = tx.prepare(STMT_INS_HEADINGS_FTS)?;
395            for doc in &changed_docs {
396                del_headings.execute(params![doc.path])?;
397                del_headings_fts.execute(params![doc.path])?;
398
399                for (level, text, anchor) in &doc.headings {
400                    ins_heading.execute(params![doc.path, level, text, anchor])?;
401                }
402
403                // Insert all heading text into FTS for search
404                if !doc.headings.is_empty() {
405                    let headings_text: Vec<&str> = doc
406                        .headings
407                        .iter()
408                        .map(|(_, text, _)| text.as_str())
409                        .collect();
410                    ins_headings_fts.execute(params![doc.path, headings_text.join(" ")])?;
411                }
412            }
413
414            // Remove files deleted from disk
415            let mut del_stale = tx.prepare("DELETE FROM documents WHERE path = ?1")?;
416            for path in &deleted {
417                del_fts.execute(params![path])?;
418                del_tags.execute(params![path])?;
419                del_headings.execute(params![path])?;
420                del_headings_fts.execute(params![path])?;
421                del_stale.execute(params![path])?;
422            }
423        }
424        tx.commit()?;
425        log::debug!(
426            "Committed transaction with {} rows, {} deleted",
427            changed_docs.len(),
428            deleted.len()
429        );
430
431        // Write back new mtimes for files whose content hash was unchanged,
432        // so subsequent refreshes don't re-read and re-hash them.
433        if !touch.is_empty() {
434            let tx = conn.unchecked_transaction()?;
435            {
436                let mut upd_meta = tx.prepare(STMT_UPD_META)?;
437                for (path, mtime, hash) in &touch {
438                    upd_meta.execute(params![path, mtime, hash])?;
439                }
440            }
441            tx.commit()?;
442            log::debug!("Wrote back mtime for {} hash-unchanged files", touch.len());
443        }
444
445        Ok(changed)
446    }
447}
grepdown_lib/db/parse.rs

grepdown_lib/db/
parse.rs
