lean_ctx/core/web/
rewrite.rs

1//! Known-host URL rewrites that turn an agent-friendly *page* URL into the URL
2//! that actually yields clean content.
3//!
4//! GitHub `blob` pages are JS-rendered: fetching them as HTML returns navigation
5//! chrome ("Star", "Fork", "Uh oh! There was an error while loading") instead of
6//! the file, and the raw HTML is enormous. The file's real bytes live on
7//! `raw.githubusercontent.com`. Rewriting `…/blob/<ref>/<path>` (and the
8//! equivalent `…/raw/<ref>/<path>`) to that host gives the agent the actual file
9//! in one bounded fetch (GH feedback: reading GitHub pages directly hangs/garbles).
10
11/// Rewrite a known page URL to its clean-content equivalent, or `None` if no
12/// rule applies (the original URL is then used unchanged).
13pub fn rewrite_url(url: &str) -> Option<String> {
14    github_blob_to_raw(url)
15}
16
17fn github_blob_to_raw(url: &str) -> Option<String> {
18    let rest = strip_github_host(url)?;
19    // Drop any #fragment (e.g. line anchors) — raw content has no anchors.
20    let path = rest.split('#').next().unwrap_or(rest);
21
22    // owner / repo / (blob|raw) / ref / path…
23    let parts: Vec<&str> = path.splitn(5, '/').collect();
24    if parts.len() != 5 {
25        return None;
26    }
27    let [owner, repo, kind, git_ref, file_path] =
28        [parts[0], parts[1], parts[2], parts[3], parts[4]];
29    if kind != "blob" && kind != "raw" {
30        return None;
31    }
32    if owner.is_empty() || repo.is_empty() || git_ref.is_empty() || file_path.is_empty() {
33        return None;
34    }
35    Some(format!(
36        "https://raw.githubusercontent.com/{owner}/{repo}/{git_ref}/{file_path}"
37    ))
38}
39
40fn strip_github_host(url: &str) -> Option<&str> {
41    const HOSTS: [&str; 3] = [
42        "https://github.com/",
43        "http://github.com/",
44        "https://www.github.com/",
45    ];
46    HOSTS.iter().find_map(|h| url.strip_prefix(h))
47}
48
49#[cfg(test)]
50mod tests {
51    use super::*;
52
53    #[test]
54    fn rewrites_blob_to_raw() {
55        assert_eq!(
56            rewrite_url("https://github.com/yvgude/lean-ctx/blob/main/README.md").as_deref(),
57            Some("https://raw.githubusercontent.com/yvgude/lean-ctx/main/README.md")
58        );
59    }
60
61    #[test]
62    fn rewrites_nested_path_and_strips_fragment() {
63        assert_eq!(
64            rewrite_url("https://github.com/o/r/blob/v1.2.3/src/core/mod.rs#L10-L20").as_deref(),
65            Some("https://raw.githubusercontent.com/o/r/v1.2.3/src/core/mod.rs")
66        );
67    }
68
69    #[test]
70    fn rewrites_raw_page_variant() {
71        assert_eq!(
72            rewrite_url("https://github.com/o/r/raw/main/a/b.txt").as_deref(),
73            Some("https://raw.githubusercontent.com/o/r/main/a/b.txt")
74        );
75    }
76
77    #[test]
78    fn leaves_repo_root_and_non_blob_untouched() {
79        // No reliable raw target without knowing the default branch.
80        assert_eq!(rewrite_url("https://github.com/o/r"), None);
81        assert_eq!(rewrite_url("https://github.com/o/r/issues/1"), None);
82        assert_eq!(rewrite_url("https://github.com/o/r/tree/main/src"), None);
83    }
84
85    #[test]
86    fn leaves_other_hosts_untouched() {
87        assert_eq!(rewrite_url("https://example.com/o/r/blob/main/x"), None);
88        assert_eq!(rewrite_url("https://gitlab.com/o/r/blob/main/x"), None);
89    }
90}
lean_ctx/core/web/rewrite.rs

lean_ctx/core/web/
rewrite.rs
