siteforge 0.1.10

Archive websites into AI-readable local knowledge archives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

pub mod archive;
pub mod cli;
pub mod config;
pub mod crawler;
pub mod errors;
pub mod export;
pub mod fetch;
pub mod ocr;
pub mod parse;
pub mod render;
pub mod search;
pub mod storage;
pub mod tui;

pub use archive::{ArchiveLayout, ArchiveManifest, CrawlStats, ThesaSidecarSummary};
pub use config::{ArchiveOutputProfile, Config};
pub use crawler::{
    ArchiveProgress, ArchiveProgressPhase, ArchiveProgressSender, ArchiveRunOptions, ArchiveSummary,
};
pub use errors::{Result, SiteforgeError};
pub use export::ExportFormat;
pub use storage::{SearchResult, StoredAsset, StoredPage};

#[derive(Debug, Clone)]
pub struct Siteforge {
    config: Config,
}

impl Siteforge {
    pub fn new(config: Config) -> Self {
        Self { config }
    }

    pub fn config(&self) -> &Config {
        &self.config
    }

    pub fn config_mut(&mut self) -> &mut Config {
        &mut self.config
    }

    pub fn archive_layout(&self, archive_id: &str) -> Result<ArchiveLayout> {
        Ok(ArchiveLayout::new(
            self.config.resolved_archive_root()?,
            archive_id.to_string(),
        ))
    }

    pub async fn archive(&self, options: ArchiveRunOptions) -> Result<ArchiveSummary> {
        crawler::run_archive(self.config.clone(), options).await
    }

    pub async fn archive_with_progress(
        &self,
        options: ArchiveRunOptions,
        progress: ArchiveProgressSender,
    ) -> Result<ArchiveSummary> {
        crawler::run_archive_with_progress(self.config.clone(), options, progress).await
    }

    pub async fn resume(&self, archive_id: &str) -> Result<ArchiveSummary> {
        crawler::resume_archive(self.config.clone(), archive_id).await
    }

    pub async fn resume_with_progress(
        &self,
        archive_id: &str,
        progress: ArchiveProgressSender,
    ) -> Result<ArchiveSummary> {
        crawler::resume_archive_with_progress(self.config.clone(), archive_id, progress).await
    }

    pub fn list_archives(&self) -> Result<Vec<ArchiveManifest>> {
        archive::list_archives(&self.config)
    }

    pub fn search(
        &self,
        archive_id: Option<&str>,
        query: &str,
        limit: usize,
    ) -> Result<Vec<SearchResult>> {
        search::search_archives(&self.config, archive_id, query, limit)
    }

    pub fn export(
        &self,
        archive_id: &str,
        format: ExportFormat,
        output: Option<&std::path::Path>,
    ) -> Result<std::path::PathBuf> {
        export::export_archive(&self.config, archive_id, format, output)
    }

    pub fn create_ai_packs(
        &self,
        archive_id: &str,
        max_tokens: Option<usize>,
    ) -> Result<Vec<std::path::PathBuf>> {
        export::create_ai_packs(&self.config, archive_id, max_tokens)
    }

    pub fn verify(&self, archive_id: &str) -> Result<Vec<String>> {
        archive::verify_archive(&self.config, archive_id)
    }
}

impl Default for Siteforge {
    fn default() -> Self {
        Self::new(Config::default())
    }
}

impl From<Config> for Siteforge {
    fn from(config: Config) -> Self {
        Self::new(config)
    }
}