webfetch/
types.rs

1use serde::{Deserialize, Serialize};
2
3pub use crate::tls::TlsConfig;
4
5/// Whether a fetch actually produced content.
6///
7/// An extraction that yields nothing used to be reported exactly like a
8/// successful one — empty `content`, exit 0 — so a caller could not tell a
9/// blank page from a page whose text never arrives without a browser. Agents
10/// read that as "this page has nothing to say" and moved on.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum ContentStatus {
14    /// Content was extracted.
15    Ok,
16    /// The document parsed but genuinely holds no text.
17    Empty,
18    /// An HTML shell with scripts and no text: the content is rendered by
19    /// JavaScript, which this fetcher does not run.
20    NeedsJs,
21    /// The document was too deeply nested to parse within a sane time budget
22    /// and was refused before parsing. See `webfetch::limits`.
23    TooComplex,
24}
25
26impl ContentStatus {
27    /// Did extraction fail to produce usable content?
28    pub fn is_failure(self) -> bool {
29        !matches!(self, ContentStatus::Ok)
30    }
31
32    /// A one-line explanation, or `None` when content came back normally.
33    pub fn note(self) -> Option<&'static str> {
34        match self {
35            ContentStatus::Ok => None,
36            ContentStatus::Empty => Some("the page parsed but contains no text"),
37            ContentStatus::NeedsJs => Some(
38                "no text content: the page renders its body with JavaScript, \
39                 which webtools does not execute",
40            ),
41            ContentStatus::TooComplex => {
42                Some("the document is too deeply nested to parse safely and was refused")
43            }
44        }
45    }
46}
47
48/// Result of fetching and converting a web page.
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct FetchResult {
51    pub title: String,
52    /// The URL the content actually came from, after redirects.
53    pub final_url: String,
54    pub content: String,
55    pub content_type: ContentType,
56    /// The detected source media kind: "html", "json", "text", or a raw
57    /// content-type for anything not rendered.
58    pub media: String,
59    pub token_estimate: usize,
60    /// Whether content was extracted — see [`ContentStatus`].
61    pub status: ContentStatus,
62    /// References cited by `content`. When `max_tokens` truncates the body,
63    /// references the surviving text no longer cites are dropped from both, so
64    /// this list and the inline `[N]` markers always agree.
65    pub references: Vec<UrlReference>,
66    #[serde(default)]
67    pub metadata: Metadata,
68    /// The URL that was requested, before any redirect.
69    pub source: String,
70}
71
72/// Citation-oriented page metadata, all best-effort.
73#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
74pub struct Metadata {
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub description: Option<String>,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub author: Option<String>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub published: Option<String>,
81    #[serde(skip_serializing_if = "Option::is_none")]
82    pub lang: Option<String>,
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub site_name: Option<String>,
85    /// The document's declared character set, when it is not UTF-8. Bodies are
86    /// decoded as UTF-8, so a value here means the text may be garbled.
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub charset: Option<String>,
89}
90
91/// A single preserved URL, recoverable by its `index`.
92#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
93pub struct UrlReference {
94    pub index: usize,
95    pub url: String,
96    /// The anchor text the link was attached to (best-effort).
97    pub text: String,
98}
99
100impl crate::refs::Referable for UrlReference {
101    fn index(&self) -> usize {
102        self.index
103    }
104    fn url(&self) -> &str {
105        &self.url
106    }
107}
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
110#[serde(rename_all = "lowercase")]
111pub enum ContentType {
112    Text,
113    Markdown,
114    Structured,
115}
116
117impl ContentType {
118    pub fn parse(s: &str) -> Self {
119        match s.to_ascii_lowercase().as_str() {
120            "markdown" | "md" => ContentType::Markdown,
121            "structured" | "json" => ContentType::Structured,
122            _ => ContentType::Text,
123        }
124    }
125}
126
127#[derive(Debug, Clone, Deserialize)]
128pub struct FetchOptions {
129    pub url: String,
130    pub content_type: ContentType,
131    pub max_tokens: Option<usize>,
132    pub timeout_secs: u64,
133    /// TLS trust configuration (OS store is honoured by default; this carries
134    /// the explicit `--ca-cert` / `--insecure` overrides).
135    #[serde(default)]
136    pub tls: TlsConfig,
137}
138
139impl Default for FetchOptions {
140    fn default() -> Self {
141        Self {
142            url: String::new(),
143            content_type: ContentType::Text,
144            max_tokens: None,
145            timeout_secs: 10,
146            tls: TlsConfig::default(),
147        }
148    }
149}
webfetch/types.rs

webfetch/
types.rs
