siteforge/
config.rs

1use std::fs;
2use std::path::{Path, PathBuf};
3
4use directories::ProjectDirs;
5use serde::{Deserialize, Serialize};
6
7use crate::errors::{Result, SiteforgeError};
8
9#[derive(Debug, Clone, Serialize, Deserialize)]
10#[serde(default)]
11/// Siteforge runtime configuration.
12///
13/// Values can be loaded from YAML with [`Config::load`] or constructed
14/// directly for library use. Relative paths resolve from the current working
15/// directory when operations run.
16pub struct Config {
17    /// Root directory where archive directories are created.
18    pub archive_root: PathBuf,
19    /// Artifact volume/profile for newly created archives.
20    pub archive_profile: ArchiveOutputProfile,
21    /// Default concurrent page fetch count.
22    pub default_concurrency: usize,
23    /// Default politeness delay between requests to the same origin, in milliseconds.
24    pub default_delay_ms: u64,
25    /// Default maximum link depth for full-site crawls.
26    pub max_depth: usize,
27    /// Default maximum number of pages to fetch per archive.
28    pub max_pages: usize,
29    /// Maximum bytes to read for a page response body. `0` means no body cap.
30    pub max_page_size_bytes: u64,
31    /// Maximum bytes to read for an asset response body. `0` means no body cap.
32    pub max_asset_size_bytes: u64,
33    /// Maximum total archive bytes. `0` disables the total archive cap.
34    pub max_total_archive_size_bytes: u64,
35    /// Optional URL glob allow-list. Empty means all in-scope URLs are eligible.
36    pub include_url_patterns: Vec<String>,
37    /// URL glob deny-list applied after scope checks.
38    pub exclude_url_patterns: Vec<String>,
39    /// Enable OCR extraction for likely code/image assets.
40    pub ocr_enabled: bool,
41    /// Shell command template for OCR. `{input}` is replaced with the asset path.
42    pub ocr_backend_command: String,
43    /// User-Agent sent by the HTTP fetcher and browser renderer.
44    pub user_agent: String,
45    /// Per-request timeout in seconds.
46    pub timeout_secs: u64,
47    /// Number of retry attempts for retryable fetch failures.
48    pub retry_count: usize,
49    /// Default token budget for generated AI context packs.
50    pub ai_pack_token_budget: usize,
51    /// Render pages with a browser before parsing when enabled.
52    pub render_js: bool,
53    /// With `render_js`, only render thin SPA-like pages instead of every page.
54    pub render_js_auto: bool,
55    /// Optional browser command or executable for JavaScript rendering.
56    pub browser_command: Option<String>,
57    /// Optional browser user-data directory/profile for authorized content.
58    pub browser_profile: Option<PathBuf>,
59    /// Browser virtual-time wait budget in milliseconds.
60    pub render_wait_ms: u64,
61}
62
63#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
64#[serde(rename_all = "kebab-case")]
65/// Controls how much artifact data each archived page writes.
66pub enum ArchiveOutputProfile {
67    /// Preserve all page artifacts: raw payloads, assets, Markdown variants, JSON, text, and chunks.
68    #[default]
69    Full,
70    /// Keep raw evidence and agent-readable files, but skip duplicate plain text and YAML Markdown.
71    Agent,
72    /// Keep compact agent-readable Markdown, JSON, chunks, metadata, and checksums only.
73    Lean,
74}
75
76impl ArchiveOutputProfile {
77    /// Whether raw fetched HTML pages are stored.
78    pub fn stores_raw_pages(self) -> bool {
79        matches!(self, Self::Full | Self::Agent)
80    }
81
82    /// Whether rendered DOM captures are stored when JS rendering succeeds.
83    pub fn stores_rendered_pages(self) -> bool {
84        matches!(self, Self::Full | Self::Agent)
85    }
86
87    /// Whether page assets are downloaded into `raw/assets/`.
88    pub fn downloads_assets(self) -> bool {
89        matches!(self, Self::Full | Self::Agent)
90    }
91
92    /// Whether YAML-frontmatter Markdown is stored in `readable/markdown/`.
93    pub fn stores_markdown(self) -> bool {
94        matches!(self, Self::Full)
95    }
96
97    /// Whether no-frontmatter basic Markdown is stored in `readable/basic_markdown/`.
98    pub fn stores_basic_markdown(self) -> bool {
99        true
100    }
101
102    /// Whether structured page JSON is stored in `readable/json/`.
103    pub fn stores_json(self) -> bool {
104        true
105    }
106
107    /// Whether plain text duplicates are stored in `readable/text/`.
108    pub fn stores_text(self) -> bool {
109        matches!(self, Self::Full)
110    }
111
112    /// Whether semantic JSONL chunks are appended to `chunks/chunks.jsonl`.
113    pub fn stores_chunks(self) -> bool {
114        true
115    }
116}
117
118impl Default for Config {
119    fn default() -> Self {
120        Self {
121            archive_root: PathBuf::from("archives"),
122            archive_profile: ArchiveOutputProfile::Full,
123            default_concurrency: 4,
124            default_delay_ms: 250,
125            max_depth: 4,
126            max_pages: 500,
127            max_page_size_bytes: 50 * 1024 * 1024,
128            max_asset_size_bytes: 25 * 1024 * 1024,
129            max_total_archive_size_bytes: 5 * 1024 * 1024 * 1024,
130            include_url_patterns: Vec::new(),
131            exclude_url_patterns: vec![
132                "**/login**".to_string(),
133                "**/logout**".to_string(),
134                "**/signup**".to_string(),
135                "**/cart**".to_string(),
136            ],
137            ocr_enabled: false,
138            ocr_backend_command: "tesseract {input} stdout".to_string(),
139            user_agent: "siteforge/0.1 (+https://github.com/Tknott95/siteforge)".to_string(),
140            timeout_secs: 30,
141            retry_count: 2,
142            ai_pack_token_budget: 120_000,
143            render_js: false,
144            render_js_auto: false,
145            browser_command: None,
146            browser_profile: None,
147            render_wait_ms: 5_000,
148        }
149    }
150}
151
152impl Config {
153    /// Load YAML configuration from `path`, or the platform default path when `None`.
154    ///
155    /// Missing files return [`Config::default`]. A configured concurrency of zero
156    /// is normalized to one.
157    pub fn load(path: Option<&Path>) -> Result<Self> {
158        let path = match path {
159            Some(path) => path.to_path_buf(),
160            None => Self::default_path()?,
161        };
162
163        if !path.exists() {
164            return Ok(Self::default());
165        }
166
167        let raw = fs::read_to_string(&path)?;
168        let mut config: Config = serde_yaml::from_str(&raw)?;
169        if config.default_concurrency == 0 {
170            config.default_concurrency = 1;
171        }
172        Ok(config)
173    }
174
175    /// Return the platform default configuration path.
176    pub fn default_path() -> Result<PathBuf> {
177        let dirs = ProjectDirs::from("com", "siteforge", "siteforge")
178            .ok_or(SiteforgeError::ConfigDirUnavailable)?;
179        Ok(dirs.config_dir().join("config.yaml"))
180    }
181
182    /// Create the platform default configuration file when missing and return its path.
183    pub fn ensure_default_file() -> Result<PathBuf> {
184        let path = Self::default_path()?;
185        if !path.exists() {
186            if let Some(parent) = path.parent() {
187                fs::create_dir_all(parent)?;
188            }
189            fs::write(&path, serde_yaml::to_string(&Self::default())?)?;
190        }
191        Ok(path)
192    }
193
194    /// Resolve [`Config::archive_root`] to an absolute path.
195    pub fn resolved_archive_root(&self) -> Result<PathBuf> {
196        if self.archive_root.is_absolute() {
197            Ok(self.archive_root.clone())
198        } else {
199            Ok(std::env::current_dir()?.join(&self.archive_root))
200        }
201    }
202}
siteforge/config.rs

siteforge/
config.rs
