agentsec_core/web/
mod.rs

1//! Web sanitize: URL fetch + 2-layer injection strip + envelope wrap.
2//! Covers the **L4 × V1** cell (cf. *crate root §Threat surface × vector*).
3//!
4//! ## Pipeline
5//!
6//! [`fetch_and_sanitize`] composes:
7//!
8//! 1. [`fetch::get`] — HTTP GET with a 2 MiB body cap and 15 s timeout.
9//!    Non-2xx responses and oversized bodies are rejected, not truncated.
10//! 2. [`sanitize::regex_layer::strip`] — fast local pass that replaces
11//!    known injection markers with the literal `[STRIPPED]`.
12//! 3. [`sanitize::semantic_layer::review`] — optional LLM-backed pass that
13//!    runs only when `ANTHROPIC_API_KEY` is set (fail-open; see that
14//!    module).
15//! 4. [`sanitize::wrap::envelope`] — wrap the cleaned body in an
16//!    `<untrusted_content src=... sanitized_at=... removed_patterns=...>`
17//!    envelope so the caller LLM physically separates instruction from
18//!    data.
19//!
20//! Result is persisted to `<home>/web_log/<UTC-ts>-<id>.json` as an audit
21//! row.
22//!
23//! ## Read-only invariant
24//!
25//! The fetched body is **never** written to disk in raw form. Only the
26//! sanitized envelope text and the list of removed patterns are persisted.
27
28pub mod fetch;
29pub mod sanitize;
30
31use crate::Config;
32use crate::error::Result;
33use serde::{Deserialize, Serialize};
34use std::fs;
35use std::path::PathBuf;
36use uuid::Uuid;
37
38/// One sanitize call's full result payload.
39#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct SanitizedContent {
41    /// UUID v4 string assigned at sanitize time.
42    pub id: String,
43    /// URL that was fetched, echoed back into the audit row and the
44    /// `<untrusted_content src=...>` attribute.
45    pub url: String,
46    /// UTC timestamp of when the fetch started.
47    pub fetched_at: chrono::DateTime<chrono::Utc>,
48    /// The wrapped `<untrusted_content>...</untrusted_content>` envelope
49    /// text, ready to hand to the caller LLM.
50    pub sanitized_text: String,
51    /// All pattern labels stripped from the body across both sanitize
52    /// layers.
53    pub removed_patterns: Vec<String>,
54    /// Audit-row path under `<home>/web_log/`.
55    pub log_path: PathBuf,
56}
57
58/// Fetch a URL, run the 2-layer sanitize pipeline, wrap in
59/// `<untrusted_content>`, and persist to `<cfg.paths.home>/web_log/`.
60///
61/// # Errors
62///
63/// - [`crate::Error::Http`] / [`crate::Error::Sanitize`] from [`fetch::get`]
64///   (network, non-2xx status, body cap exceeded).
65/// - [`crate::Error::Io`] / [`crate::Error::Json`] from log persistence.
66///
67/// The semantic sanitize layer fails open: a missing
68/// `ANTHROPIC_API_KEY` (i.e. `cfg.llm.api_key == None`) or non-2xx API
69/// response returns the regex-stripped text unchanged, so an LLM outage
70/// does not break URL fetch.
71pub async fn fetch_and_sanitize(cfg: &Config, url: &str) -> Result<SanitizedContent> {
72    let raw = fetch::get(url).await?;
73    let (cleaned_text, removed) = sanitize::run(&cfg.llm, url, &raw).await?;
74
75    let mut content = SanitizedContent {
76        id: Uuid::new_v4().to_string(),
77        url: url.to_string(),
78        fetched_at: chrono::Utc::now(),
79        sanitized_text: cleaned_text,
80        removed_patterns: removed,
81        log_path: PathBuf::new(),
82    };
83
84    // Persist on best-effort basis (same self-referential-path pattern as
85    // `paste::detect`): compute the path, assign to `content.log_path`,
86    // then serialize so the JSON on disk carries the same path the
87    // in-memory return value does.
88    persist_log(&cfg.paths, &mut content)?;
89    Ok(content)
90}
91
92fn persist_log(paths: &crate::Paths, content: &mut SanitizedContent) -> Result<()> {
93    let dir = paths.web_log();
94    fs::create_dir_all(&dir)?;
95    let stamp = content.fetched_at.format("%Y-%m-%d-%H%M%S");
96    let path = dir.join(format!("{stamp}-{}.json", &content.id[..8]));
97    content.log_path.clone_from(&path);
98    let body = serde_json::to_string_pretty(content)?;
99    fs::write(&path, body)?;
100    Ok(())
101}
agentsec_core/web/mod.rs

agentsec_core/web/
mod.rs
