agentsec_core/web/
mod.rs

1//! Web sanitize: URL fetch + 2-layer injection strip + envelope wrap.
2//! Covers the **L4 × V1** cell (cf. *crate root §Threat surface × vector*).
3//!
4//! ## Pipeline
5//!
6//! [`fetch_and_sanitize`] composes:
7//!
8//! 1. [`fetch::get`] — HTTP GET with a 2 MiB body cap and a configurable timeout
9//!    driven by [`crate::config::WebConfig::timeout_secs`].
10//!    Non-2xx responses and oversized bodies are rejected, not truncated.
11//! 2. [`sanitize::regex_layer::strip`] — fast local pass that replaces
12//!    known injection markers with the literal `[STRIPPED]`.
13//! 3. [`sanitize::semantic_layer::review`] — optional LLM-backed pass that
14//!    runs only when `ANTHROPIC_API_KEY` is set (fail-open; see that
15//!    module).
16//! 4. [`sanitize::wrap::envelope`] — wrap the cleaned body in an
17//!    `<untrusted_content src=... sanitized_at=... removed_patterns=...>`
18//!    envelope so the caller LLM physically separates instruction from
19//!    data.
20//!
21//! Result is persisted to `<home>/web_log/<UTC-ts>-<id>.json` as an audit
22//! row.
23//!
24//! ## Read-only invariant
25//!
26//! The fetched body is **never** written to disk in raw form. Only the
27//! sanitized envelope text and the list of removed patterns are persisted.
28
29pub mod fetch;
30pub mod sanitize;
31
32use crate::Config;
33use crate::error::Result;
34use serde::{Deserialize, Serialize};
35use std::fs;
36use std::path::PathBuf;
37use uuid::Uuid;
38
39/// One sanitize call's full result payload.
40#[derive(Debug, Clone, Serialize, Deserialize)]
41pub struct SanitizedContent {
42    /// UUID v4 string assigned at sanitize time.
43    pub id: String,
44    /// URL that was fetched, echoed back into the audit row and the
45    /// `<untrusted_content src=...>` attribute.
46    pub url: String,
47    /// UTC timestamp of when the fetch started.
48    pub fetched_at: chrono::DateTime<chrono::Utc>,
49    /// The wrapped `<untrusted_content>...</untrusted_content>` envelope
50    /// text, ready to hand to the caller LLM.
51    pub sanitized_text: String,
52    /// All pattern labels stripped from the body across both sanitize
53    /// layers.
54    pub removed_patterns: Vec<String>,
55    /// Audit-row path under `<home>/web_log/`.
56    pub log_path: PathBuf,
57}
58
59/// Fetch a URL, run the 2-layer sanitize pipeline, wrap in
60/// `<untrusted_content>`, and persist to `<cfg.paths.home>/web_log/`.
61///
62/// # Errors
63///
64/// - [`crate::Error::Http`] / [`crate::Error::Sanitize`] from [`fetch::get`]
65///   (network, non-2xx status, body cap exceeded).
66/// - [`crate::Error::Io`] / [`crate::Error::Json`] from log persistence.
67///
68/// The semantic sanitize layer fails open: a missing
69/// `ANTHROPIC_API_KEY` (i.e. `cfg.llm.api_key == None`) or non-2xx API
70/// response returns the regex-stripped text unchanged, so an LLM outage
71/// does not break URL fetch.
72pub async fn fetch_and_sanitize(cfg: &Config, url: &str) -> Result<SanitizedContent> {
73    let raw = fetch::get(url, cfg.web.timeout_secs).await?;
74    let (cleaned_text, removed) = sanitize::run(&cfg.llm, url, &raw).await?;
75
76    let mut content = SanitizedContent {
77        id: Uuid::new_v4().to_string(),
78        url: url.to_string(),
79        fetched_at: chrono::Utc::now(),
80        sanitized_text: cleaned_text,
81        removed_patterns: removed,
82        log_path: PathBuf::new(),
83    };
84
85    // Persist on best-effort basis (same self-referential-path pattern as
86    // `paste::detect`): compute the path, assign to `content.log_path`,
87    // then serialize so the JSON on disk carries the same path the
88    // in-memory return value does.
89    persist_log(&cfg.paths, &mut content)?;
90    Ok(content)
91}
92
93fn persist_log(paths: &crate::Paths, content: &mut SanitizedContent) -> Result<()> {
94    let dir = paths.web_log();
95    fs::create_dir_all(&dir)?;
96    let stamp = content.fetched_at.format("%Y-%m-%d-%H%M%S");
97    let path = dir.join(format!("{stamp}-{}.json", &content.id[..8]));
98    content.log_path.clone_from(&path);
99    let body = serde_json::to_string_pretty(content)?;
100    fs::write(&path, body)?;
101    Ok(())
102}
agentsec_core/web/mod.rs

agentsec_core/web/
mod.rs
