siteforge 0.1.4

Archive websites into AI-readable local knowledge archives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

use std::path::PathBuf;
use std::process::Command;

use url::Url;

use crate::config::Config;
use crate::errors::{Result, SiteforgeError};

#[derive(Debug, Clone)]
pub struct BrowserRenderOptions {
    pub command: Option<String>,
    pub profile: Option<PathBuf>,
    pub wait_ms: u64,
    pub user_agent: String,
}

impl BrowserRenderOptions {
    pub fn from_config(config: &Config) -> Self {
        Self {
            command: config.browser_command.clone(),
            profile: config.browser_profile.clone(),
            wait_ms: config.render_wait_ms,
            user_agent: config.user_agent.clone(),
        }
    }
}

pub async fn render_js_dom(options: BrowserRenderOptions, url: Url) -> Result<String> {
    tokio::task::spawn_blocking(move || render_js_dom_blocking(&options, &url))
        .await
        .map_err(|err| SiteforgeError::message(format!("browser render task failed: {err}")))?
}

fn render_js_dom_blocking(options: &BrowserRenderOptions, url: &Url) -> Result<String> {
    if let Some(command) = options.command.as_deref() {
        return run_command(command, options, url);
    }

    let candidates = [
        "chromium",
        "chromium-browser",
        "google-chrome",
        "google-chrome-stable",
    ];
    let mut errors = Vec::new();
    for candidate in candidates {
        match run_browser(candidate, &[], options, url) {
            Ok(html) => return Ok(html),
            Err(err) => errors.push(format!("{candidate}: {err}")),
        }
    }
    Err(SiteforgeError::message(format!(
        "no working browser renderer found; set browser_command. Tried: {}",
        errors.join("; ")
    )))
}

fn run_command(command: &str, options: &BrowserRenderOptions, url: &Url) -> Result<String> {
    let rendered = command
        .replace("{url}", url.as_str())
        .replace("{wait_ms}", &options.wait_ms.to_string())
        .replace("{user_agent}", &options.user_agent);
    let parts = shell_words::split(&rendered)
        .map_err(|err| SiteforgeError::message(format!("invalid browser command: {err}")))?;
    let (program, args) = parts
        .split_first()
        .ok_or_else(|| SiteforgeError::message("empty browser command"))?;

    if command.contains("{url}") {
        run_raw_command(program, args)
    } else {
        run_browser(program, args, options, url)
    }
}

fn run_browser(
    program: &str,
    prefix_args: &[String],
    options: &BrowserRenderOptions,
    url: &Url,
) -> Result<String> {
    let mut args = prefix_args.to_vec();
    args.extend([
        "--headless".to_string(),
        "--disable-gpu".to_string(),
        "--no-sandbox".to_string(),
        "--disable-dev-shm-usage".to_string(),
        "--dump-dom".to_string(),
        format!("--virtual-time-budget={}", options.wait_ms),
        format!("--user-agent={}", options.user_agent),
    ]);
    if let Some(profile) = options.profile.as_deref() {
        args.push(format!("--user-data-dir={}", profile.display()));
    }
    args.push(url.to_string());
    run_raw_command(program, &args)
}

fn run_raw_command(program: &str, args: &[String]) -> Result<String> {
    let output = Command::new(program).args(args).output().map_err(|err| {
        SiteforgeError::message(format!("failed to start browser renderer {program}: {err}"))
    })?;
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        return Err(SiteforgeError::message(format!(
            "browser renderer exited with {}: {}",
            output.status, stderr
        )));
    }
    let html = String::from_utf8_lossy(&output.stdout).trim().to_string();
    if html.is_empty() {
        return Err(SiteforgeError::message(
            "browser renderer returned empty HTML".to_string(),
        ));
    }
    Ok(html)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn custom_command_template_replaces_values() {
        let options = BrowserRenderOptions {
            command: Some("printf '<html>{url}:{wait_ms}</html>'".to_string()),
            profile: None,
            wait_ms: 42,
            user_agent: "agent".to_string(),
        };
        let url = Url::parse("https://example.test/").unwrap();
        let html = render_js_dom_blocking(&options, &url).unwrap();
        assert!(html.contains("https://example.test/:42"));
    }
}