siteforge 0.1.4

Archive websites into AI-readable local knowledge archives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use std::collections::HashSet;
use std::fs;

use crate::archive::{list_archives, ArchiveLayout};
use crate::config::Config;
use crate::errors::{Result, SiteforgeError};
use crate::storage::{SearchResult, Storage};

pub fn search_archives(
    config: &Config,
    archive_id: Option<&str>,
    query: &str,
    limit: usize,
) -> Result<Vec<SearchResult>> {
    let root = config.resolved_archive_root()?;
    let archive_ids = match archive_id {
        Some(id) => vec![id.to_string()],
        None => list_archives(config)?
            .into_iter()
            .map(|manifest| manifest.archive_id)
            .collect(),
    };

    let mut results = Vec::new();
    for id in archive_ids {
        let layout = ArchiveLayout::new(root.clone(), id.clone());
        if !layout.database().exists() {
            if archive_id.is_some() {
                return Err(SiteforgeError::ArchiveNotFound(id));
            }
            continue;
        }
        let storage = Storage::open(&layout.database())?;
        let mut seen_pages = HashSet::new();
        for mut result in storage.search(query, limit)? {
            if let Some(path) = &result.local_file_path {
                result.local_file_path = Some(layout.base().join(path).display().to_string());
            }
            seen_pages.insert(result.page_id.clone());
            results.push(result);
        }
        if results.len() < limit {
            results.extend(search_readable_files(
                &id,
                &layout,
                &storage,
                query,
                limit - results.len(),
                &seen_pages,
            )?);
        }
    }
    results.truncate(limit);
    Ok(results)
}

fn search_readable_files(
    archive_id: &str,
    layout: &ArchiveLayout,
    storage: &Storage,
    query: &str,
    limit: usize,
    seen_pages: &HashSet<String>,
) -> Result<Vec<SearchResult>> {
    let query_lower = query.to_ascii_lowercase();
    let mut results = Vec::new();
    for page in storage.list_pages()? {
        if seen_pages.contains(&page.page_id) {
            continue;
        }
        let readable_path = page
            .text_path
            .as_deref()
            .or(page.markdown_path.as_deref())
            .map(|path| layout.base().join(path));
        let Some(path) = readable_path else {
            continue;
        };
        let Ok(text) = fs::read_to_string(&path) else {
            continue;
        };
        if !text.to_ascii_lowercase().contains(&query_lower) {
            continue;
        }
        results.push(SearchResult {
            archive_id: archive_id.to_string(),
            page_id: page.page_id,
            source_url: page.source_url,
            title: page.title,
            matched_heading: None,
            snippet: file_snippet(&text, query),
            local_file_path: Some(path.display().to_string()),
        });
        if results.len() >= limit {
            break;
        }
    }
    Ok(results)
}

fn file_snippet(text: &str, query: &str) -> String {
    let lower = text.to_ascii_lowercase();
    let query_lower = query.to_ascii_lowercase();
    let pos = lower.find(&query_lower).unwrap_or(0);
    let start = pos.saturating_sub(80);
    let end = (pos + query.len() + 160).min(text.len());
    text.get(start..end).unwrap_or(text).replace('\n', " ")
}