webfind 0.5.0

Self-hosted native search engine for AI agents — pure CLI, zero cost
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

//! Tier-2 Metadata Sniffer: detects machine-readable agent endpoints.
//!
//! Checks domains for machine-readable manifests (`/.well-known/ai-catalog.json`,
//! `/llms.txt`, `/llms-full.txt`, `/openapi.json`, `/swagger.json`) before falling
//! back to full DOM scraping or headless Chromium rendering.

use reqwest::Client;
use serde::{Deserialize, Serialize};
use url::Url;

/// Machine-readable affordances discovered on a domain.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct MachineAffordances {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub llms_txt: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub ai_catalog: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub openapi_spec: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mcp_endpoint: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub ax_score: Option<f64>,
}

impl MachineAffordances {
    /// True when at least one machine-readable affordance is present.
    #[must_use]
    pub fn has_affordances(&self) -> bool {
        self.llms_txt.is_some()
            || self.ai_catalog.is_some()
            || self.openapi_spec.is_some()
            || self.mcp_endpoint.is_some()
    }
}

/// Metadata sniffer for agentic resource discovery.
pub struct MetadataSniffer {
    client: Client,
}

impl MetadataSniffer {
    /// Create a new metadata sniffer with the shared HTTP client.
    #[must_use]
    pub fn new(client: Client) -> Self {
        Self { client }
    }

    /// Probe a domain for agent-native metadata endpoints.
    pub async fn sniff_domain(&self, base_url: &str) -> MachineAffordances {
        let Ok(parsed) = Url::parse(base_url) else {
            return MachineAffordances::default();
        };

        let origin = match (parsed.scheme(), parsed.host_str(), parsed.port()) {
            (scheme, Some(host), Some(port)) => format!("{scheme}://{host}:{port}"),
            (scheme, Some(host), None) => format!("{scheme}://{host}"),
            _ => return MachineAffordances::default(),
        };

        let mut affordances = MachineAffordances::default();

        // 1. Probe /llms-full.txt or /llms.txt
        if let Ok(full_url) = Url::parse(&format!("{origin}/llms-full.txt"))
            && self.probe_endpoint(&full_url).await
        {
            affordances.llms_txt = Some(full_url.to_string());
        } else if let Ok(url) = Url::parse(&format!("{origin}/llms.txt"))
            && self.probe_endpoint(&url).await
        {
            affordances.llms_txt = Some(url.to_string());
        }

        // 2. Probe AI catalog variants: /.well-known/ai-catalog.json, /.well-known/ai-catelog.json, /ai-catalog.json, /ai-catelog.json
        let catalog_candidates = [
            format!("{origin}/.well-known/ai-catalog.json"),
            format!("{origin}/.well-known/ai-catelog.json"),
            format!("{origin}/ai-catalog.json"),
            format!("{origin}/ai-catelog.json"),
        ];
        for cand in &catalog_candidates {
            if let Ok(url) = Url::parse(cand)
                && self.probe_endpoint(&url).await
            {
                affordances.ai_catalog = Some(url.to_string());
                break;
            }
        }

        // 3. Probe /openapi.json
        if let Ok(url) = Url::parse(&format!("{origin}/openapi.json"))
            && self.probe_endpoint(&url).await
        {
            affordances.openapi_spec = Some(url.to_string());
        }

        affordances
    }

    /// Probe an endpoint with a lightweight HEAD/GET request.
    async fn probe_endpoint(&self, url: &Url) -> bool {
        // Fast timeout for sniffer probes (1.5s max)
        let resp = self
            .client
            .head(url.as_str())
            .timeout(std::time::Duration::from_millis(1500))
            .send()
            .await;

        if let Ok(res) = resp
            && res.status().is_success()
        {
            return true;
        }

        // Fallback to GET with small range if HEAD is blocked by origin
        let get_resp = self
            .client
            .get(url.as_str())
            .header(reqwest::header::RANGE, "bytes=0-1024")
            .timeout(std::time::Duration::from_millis(1500))
            .send()
            .await;

        if let Ok(res) = get_resp {
            res.status().is_success() || res.status() == reqwest::StatusCode::PARTIAL_CONTENT
        } else {
            false
        }
    }

    /// Fetch the full content of an affordance endpoint (e.g. /llms-full.txt or /llms.txt) as clean text.
    pub async fn fetch_manifest_content(&self, url_str: &str) -> Option<String> {
        let resp = self
            .client
            .get(url_str)
            .timeout(std::time::Duration::from_millis(3500))
            .send()
            .await
            .ok()?;

        if resp.status().is_success() {
            let body = resp.text().await.ok()?;
            if !body.trim().is_empty() && !body.to_ascii_lowercase().contains("<html") {
                return Some(body);
            }
        }
        None
    }
}