agentsec-core 0.5.0

//! Web sanitize: URL fetch + 2-layer injection strip + envelope wrap.
//! Covers the **L4 × V1** cell (cf. *crate root §Threat surface × vector*).
//!
//! ## Pipeline
//!
//! [`fetch_and_sanitize`] composes:
//!
//! 1. [`fetch::get`] — HTTP GET with a 2 MiB body cap and a configurable timeout
//!    driven by [`crate::config::WebConfig::timeout_secs`].
//!    Non-2xx responses and oversized bodies are rejected, not truncated.
//! 2. [`sanitize::regex_layer::strip`] — fast local pass that replaces
//!    known injection markers with the literal `[STRIPPED]`.
//! 3. [`sanitize::semantic_layer::review`] — optional LLM-backed pass that
//!    runs only when `ANTHROPIC_API_KEY` is set (fail-open; see that
//!    module).
//! 4. [`sanitize::wrap::envelope`] — wrap the cleaned body in an
//!    `<untrusted_content src=... sanitized_at=... removed_patterns=...>`
//!    envelope so the caller LLM physically separates instruction from
//!    data.
//!
//! Result is persisted to `<home>/web_log/<UTC-ts>-<id>.json` as an audit
//! row.
//!
//! ## Read-only invariant
//!
//! The fetched body is **never** written to disk in raw form. Only the
//! sanitized envelope text and the list of removed patterns are persisted.

pub mod fetch;
pub mod sanitize;

use crate::Config;
use crate::error::Result;
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::PathBuf;
use uuid::Uuid;

/// One sanitize call's full result payload.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SanitizedContent {
    /// UUID v4 string assigned at sanitize time.
    pub id: String,
    /// URL that was fetched, echoed back into the audit row and the
    /// `<untrusted_content src=...>` attribute.
    pub url: String,
    /// UTC timestamp of when the fetch started.
    pub fetched_at: chrono::DateTime<chrono::Utc>,
    /// The wrapped `<untrusted_content>...</untrusted_content>` envelope
    /// text, ready to hand to the caller LLM.
    pub sanitized_text: String,
    /// All pattern labels stripped from the body across both sanitize
    /// layers.
    pub removed_patterns: Vec<String>,
    /// Audit-row path under `<home>/web_log/`.
    pub log_path: PathBuf,
}

/// Fetch a URL, run the 2-layer sanitize pipeline, wrap in
/// `<untrusted_content>`, and persist to `<cfg.paths.home>/web_log/`.
///
/// # Errors
///
/// - [`crate::Error::Http`] / [`crate::Error::Sanitize`] from [`fetch::get`]
///   (network, non-2xx status, body cap exceeded).
/// - [`crate::Error::Io`] / [`crate::Error::Json`] from log persistence.
///
/// The semantic sanitize layer fails open: a missing
/// `ANTHROPIC_API_KEY` (i.e. `cfg.llm.api_key == None`) or non-2xx API
/// response returns the regex-stripped text unchanged, so an LLM outage
/// does not break URL fetch.
pub async fn fetch_and_sanitize(cfg: &Config, url: &str) -> Result<SanitizedContent> {
    let raw = fetch::get(url, cfg.web.timeout_secs).await?;
    let (cleaned_text, removed) = sanitize::run(&cfg.llm, url, &raw).await?;

    let mut content = SanitizedContent {
        id: Uuid::new_v4().to_string(),
        url: url.to_string(),
        fetched_at: chrono::Utc::now(),
        sanitized_text: cleaned_text,
        removed_patterns: removed,
        log_path: PathBuf::new(),
    };

    // Persist on best-effort basis (same self-referential-path pattern as
    // `paste::detect`): compute the path, assign to `content.log_path`,
    // then serialize so the JSON on disk carries the same path the
    // in-memory return value does.
    persist_log(&cfg.paths, &mut content)?;
    Ok(content)
}

fn persist_log(paths: &crate::Paths, content: &mut SanitizedContent) -> Result<()> {
    let dir = paths.web_log();
    fs::create_dir_all(&dir)?;
    let stamp = content.fetched_at.format("%Y-%m-%d-%H%M%S");
    let path = dir.join(format!("{stamp}-{}.json", &content.id[..8]));
    content.log_path.clone_from(&path);
    let body = serde_json::to_string_pretty(content)?;
    fs::write(&path, body)?;
    Ok(())
}