blz_core/
url_resolver.rs

1//! URL resolution utilities for smart llms.txt variant detection.
2//!
3//! Automatically prefers llms-full.txt over llms.txt when available, with
4//! fallback to exact URL if neither variant exists. Uses HEAD requests to
5//! check availability before fetching content.
6
7use std::fmt::Write as _;
8
9use tracing::{debug, warn};
10
11use crate::{ContentType, Error, Fetcher, Result, SourceVariant};
12
13/// Result of URL resolution with variant and content info.
14#[derive(Debug, Clone)]
15pub struct ResolvedUrl {
16    /// The final URL that succeeded.
17    pub final_url: String,
18    /// Which variant was resolved.
19    pub variant: SourceVariant,
20    /// Content type based on line count.
21    pub content_type: ContentType,
22    /// Number of lines in the content.
23    pub line_count: usize,
24    /// Whether to warn the user about this source.
25    pub should_warn: bool,
26}
27
28/// Resolve the best URL for llms.txt documentation.
29///
30/// Tries URLs in order:
31/// 1. llms-full.txt variant (if base URL ends in .txt)
32/// 2. llms.txt variant (if base URL ends in -full.txt)
33/// 3. Exact URL as provided
34///
35/// Uses HEAD requests to check availability before fetching content.
36pub async fn resolve_best_url(fetcher: &Fetcher, base_url: &str) -> Result<ResolvedUrl> {
37    let variants = [
38        try_full_variant(base_url),
39        Some(base_url.to_string()),
40        try_base_variant(base_url),
41    ];
42    let mut blocked_statuses: Vec<u16> = Vec::new();
43    let mut rejected_statuses: Vec<u16> = Vec::new();
44
45    for (idx, maybe_url) in variants.iter().enumerate() {
46        if let Some(url) = maybe_url {
47            let should_fetch = match fetcher.head_metadata(url).await {
48                Ok(head_info) => {
49                    let status = head_info.status;
50                    if (200..=399).contains(&status) {
51                        true
52                    } else if status == 405 || status == 501 {
53                        warn!(
54                            %status,
55                            %url,
56                            "HEAD not supported; falling back to GET for candidate URL"
57                        );
58                        true
59                    } else {
60                        if matches!(status, 401 | 403 | 429) {
61                            blocked_statuses.push(status);
62                        } else {
63                            rejected_statuses.push(status);
64                        }
65                        debug!(
66                            %status,
67                            %url,
68                            "HEAD preflight rejected candidate URL"
69                        );
70                        false
71                    }
72                },
73                Err(err) => {
74                    debug!(error = %err, %url, "HEAD preflight failed for candidate URL");
75                    false
76                },
77            };
78
79            if !should_fetch {
80                continue;
81            }
82
83            let (content, _sha256) = match fetcher.fetch(url).await {
84                Ok(result) => result,
85                Err(err) => {
86                    debug!(error = %err, %url, "GET fallback failed for candidate URL");
87                    continue;
88                },
89            };
90
91            let line_count = content.lines().count();
92            let (content_type, should_warn) = classify_content(line_count);
93
94            let variant = match idx {
95                0 => SourceVariant::LlmsFull,
96                1 => {
97                    if base_url.ends_with("llms-full.txt") {
98                        SourceVariant::LlmsFull
99                    } else if base_url.ends_with("llms.txt") {
100                        SourceVariant::Llms
101                    } else {
102                        SourceVariant::Custom
103                    }
104                },
105                2 => SourceVariant::Llms,
106                _ => SourceVariant::Custom,
107            };
108
109            return Ok(ResolvedUrl {
110                final_url: url.clone(),
111                variant,
112                content_type,
113                line_count,
114                should_warn,
115            });
116        }
117    }
118
119    Err(Error::NotFound(build_resolution_error_message(
120        base_url,
121        &blocked_statuses,
122        &rejected_statuses,
123    )))
124}
125
126fn build_resolution_error_message(
127    base_url: &str,
128    blocked_statuses: &[u16],
129    rejected_statuses: &[u16],
130) -> String {
131    let mut message = String::new();
132    let _ = write!(
133        message,
134        "Failed to resolve any llms.txt variant for '{base_url}'. \
135         Tried: llms-full.txt, exact URL, and llms.txt fallback."
136    );
137
138    if !blocked_statuses.is_empty() {
139        message.push_str(" HEAD returned ");
140        write_status_list(&mut message, blocked_statuses);
141        message.push_str(
142            "; the host may be blocking automated requests. \
143             Try a direct llms.txt/llms-full.txt URL or add a local file via a manifest.",
144        );
145    } else if !rejected_statuses.is_empty() {
146        message.push_str(" HEAD returned non-success status codes (");
147        write_status_list(&mut message, rejected_statuses);
148        message.push_str(").");
149    }
150
151    message
152}
153
154fn write_status_list(message: &mut String, statuses: &[u16]) {
155    for (idx, status) in statuses.iter().enumerate() {
156        if idx > 0 {
157            message.push_str(", ");
158        }
159        let _ = write!(message, "{status}");
160    }
161}
162
163fn try_full_variant(url: &str) -> Option<String> {
164    if url.ends_with("llms.txt") && !url.ends_with("llms-full.txt") {
165        Some(url.replace("llms.txt", "llms-full.txt"))
166    } else {
167        None
168    }
169}
170
171fn try_base_variant(url: &str) -> Option<String> {
172    if url.ends_with("llms-full.txt") {
173        Some(url.replace("llms-full.txt", "llms.txt"))
174    } else {
175        None
176    }
177}
178
179const fn classify_content(line_count: usize) -> (ContentType, bool) {
180    if line_count > 1000 {
181        (ContentType::Full, false)
182    } else if line_count < 100 {
183        (ContentType::Index, true)
184    } else {
185        (ContentType::Mixed, false)
186    }
187}
188
189#[cfg(test)]
190mod tests {
191    use super::*;
192
193    #[test]
194    fn test_try_full_variant() {
195        assert_eq!(
196            try_full_variant("https://example.com/llms.txt"),
197            Some("https://example.com/llms-full.txt".to_string())
198        );
199
200        assert_eq!(try_full_variant("https://example.com/llms-full.txt"), None);
201
202        assert_eq!(try_full_variant("https://example.com/docs.txt"), None);
203    }
204
205    #[test]
206    fn test_try_base_variant() {
207        assert_eq!(
208            try_base_variant("https://example.com/llms-full.txt"),
209            Some("https://example.com/llms.txt".to_string())
210        );
211
212        assert_eq!(try_base_variant("https://example.com/llms.txt"), None);
213
214        assert_eq!(try_base_variant("https://example.com/docs.txt"), None);
215    }
216}
blz_core/url_resolver.rs

blz_core/
url_resolver.rs
