use std::path::Path;
use crate::error::{Result, SearchError};
use crate::parser::ParsedDoc;
use crate::parser::walker::FileKind;
use crate::indexer::schema::StructuralMeta;
pub fn parse_text(path: &Path, kind: &FileKind) -> Result<ParsedDoc> {
let raw = read_with_fallback(path)?;
let (text, title) = match kind {
FileKind::Markdown => extract_markdown(raw),
_ => (raw, None),
};
let snippet = text.chars().take(256).collect();
let metadata = StructuralMeta::default();
Ok(ParsedDoc {
text,
title,
snippet,
metadata,
})
}
fn read_with_fallback(path: &Path) -> Result<String> {
let bytes = std::fs::read(path).map_err(|e| SearchError::Parse {
path: path.to_path_buf(),
reason: e.to_string(),
})?;
let bytes = if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
&bytes[3..]
} else {
&bytes
};
match std::str::from_utf8(bytes) {
Ok(s) => Ok(s.to_string()),
Err(_) => {
Ok(bytes.iter().map(|&b| b as char).collect())
}
}
}
fn extract_markdown(raw: String) -> (String, Option<String>) {
let mut text = raw.as_str();
let mut title = None;
if text.starts_with("---") {
if let Some(end) = text[3..].find("\n---") {
text = &text[3 + end + 4..];
}
}
for line in text.lines() {
let line = line.trim();
if let Some(heading) = line.strip_prefix("# ") {
title = Some(heading.trim().to_string());
break;
}
}
let cleaned = strip_markdown_syntax(text);
(cleaned, title)
}
fn strip_markdown_syntax(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for line in s.lines() {
let line = line.trim_start_matches('#').trim();
let line = line.replace("**", "").replace("__", "").replace('`', "");
let line = strip_md_links(&line);
out.push_str(&line);
out.push('\n');
}
out
}
fn strip_md_links(s: &str) -> String {
let mut out = String::new();
let mut rest = s;
while let Some(start) = rest.find('[') {
out.push_str(&rest[..start]);
rest = &rest[start + 1..];
if let Some(end_bracket) = rest.find("](") {
let link_text = &rest[..end_bracket];
rest = &rest[end_bracket + 2..];
if let Some(end_paren) = rest.find(')') {
out.push_str(link_text);
rest = &rest[end_paren + 1..];
} else {
out.push('[');
out.push_str(link_text);
}
} else {
out.push('[');
}
}
out.push_str(rest);
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_frontmatter_stripped() {
let md = "---\ntitle: Test\n---\n# Hello\nWorld".to_string();
let (text, title) = extract_markdown(md);
assert_eq!(title, Some("Hello".to_string()));
assert!(text.contains("World"));
assert!(!text.contains("title: Test"));
}
#[test]
fn test_md_links_stripped() {
let result = strip_md_links("See [docs](https://example.com) for details");
assert_eq!(result, "See docs for details");
}
}