siteforge 0.1.11

Archive websites into AI-readable local knowledge archives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

use std::fs;
use std::path::{Path, PathBuf};

use directories::ProjectDirs;
use serde::{Deserialize, Serialize};

use crate::errors::{Result, SiteforgeError};

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct Config {
    pub archive_root: PathBuf,
    pub archive_profile: ArchiveOutputProfile,
    pub default_concurrency: usize,
    pub default_delay_ms: u64,
    pub max_depth: usize,
    pub max_pages: usize,
    pub max_page_size_bytes: u64,
    pub max_asset_size_bytes: u64,
    pub max_total_archive_size_bytes: u64,
    pub include_url_patterns: Vec<String>,
    pub exclude_url_patterns: Vec<String>,
    pub ocr_enabled: bool,
    pub ocr_backend_command: String,
    pub user_agent: String,
    pub timeout_secs: u64,
    pub retry_count: usize,
    pub ai_pack_token_budget: usize,
    pub render_js: bool,
    pub render_js_auto: bool,
    pub browser_command: Option<String>,
    pub browser_profile: Option<PathBuf>,
    pub render_wait_ms: u64,
}

#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub enum ArchiveOutputProfile {
    #[default]
    Full,
    Agent,
    Lean,
}

impl ArchiveOutputProfile {
    pub fn stores_raw_pages(self) -> bool {
        matches!(self, Self::Full | Self::Agent)
    }

    pub fn stores_rendered_pages(self) -> bool {
        matches!(self, Self::Full | Self::Agent)
    }

    pub fn downloads_assets(self) -> bool {
        matches!(self, Self::Full | Self::Agent)
    }

    pub fn stores_markdown(self) -> bool {
        matches!(self, Self::Full)
    }

    pub fn stores_basic_markdown(self) -> bool {
        true
    }

    pub fn stores_json(self) -> bool {
        true
    }

    pub fn stores_text(self) -> bool {
        matches!(self, Self::Full)
    }

    pub fn stores_chunks(self) -> bool {
        true
    }
}

impl Default for Config {
    fn default() -> Self {
        Self {
            archive_root: PathBuf::from("archives"),
            archive_profile: ArchiveOutputProfile::Full,
            default_concurrency: 4,
            default_delay_ms: 250,
            max_depth: 4,
            max_pages: 500,
            max_page_size_bytes: 50 * 1024 * 1024,
            max_asset_size_bytes: 25 * 1024 * 1024,
            max_total_archive_size_bytes: 5 * 1024 * 1024 * 1024,
            include_url_patterns: Vec::new(),
            exclude_url_patterns: vec![
                "**/login**".to_string(),
                "**/logout**".to_string(),
                "**/signup**".to_string(),
                "**/cart**".to_string(),
            ],
            ocr_enabled: false,
            ocr_backend_command: "tesseract {input} stdout".to_string(),
            user_agent: "siteforge/0.1 (+https://github.com/Tknott95/siteforge)".to_string(),
            timeout_secs: 30,
            retry_count: 2,
            ai_pack_token_budget: 120_000,
            render_js: false,
            render_js_auto: false,
            browser_command: None,
            browser_profile: None,
            render_wait_ms: 5_000,
        }
    }
}

impl Config {
    pub fn load(path: Option<&Path>) -> Result<Self> {
        let path = match path {
            Some(path) => path.to_path_buf(),
            None => Self::default_path()?,
        };

        if !path.exists() {
            return Ok(Self::default());
        }

        let raw = fs::read_to_string(&path)?;
        let mut config: Config = serde_yaml::from_str(&raw)?;
        if config.default_concurrency == 0 {
            config.default_concurrency = 1;
        }
        Ok(config)
    }

    pub fn default_path() -> Result<PathBuf> {
        let dirs = ProjectDirs::from("com", "siteforge", "siteforge")
            .ok_or(SiteforgeError::ConfigDirUnavailable)?;
        Ok(dirs.config_dir().join("config.yaml"))
    }

    pub fn ensure_default_file() -> Result<PathBuf> {
        let path = Self::default_path()?;
        if !path.exists() {
            if let Some(parent) = path.parent() {
                fs::create_dir_all(parent)?;
            }
            fs::write(&path, serde_yaml::to_string(&Self::default())?)?;
        }
        Ok(path)
    }

    pub fn resolved_archive_root(&self) -> Result<PathBuf> {
        if self.archive_root.is_absolute() {
            Ok(self.archive_root.clone())
        } else {
            Ok(std::env::current_dir()?.join(&self.archive_root))
        }
    }
}