Skip to main content

uxn_tal/fetch/
github.rs

1use super::{
2    downloader::{http_get, write_bytes},
3    includes::{parse_includes, resolve_include},
4    provider::{FetchResult, Provider, RepoRef},
5};
6use std::{
7    collections::{HashSet, VecDeque},
8    fs,
9    path::{Path, PathBuf},
10};
11use url::Url;
12
13pub struct GitHub;
14
15impl GitHub {
16    fn raw_url(r: &RepoRef, repo_rel: &str) -> String {
17        format!(
18            "https://raw.githubusercontent.com/{}/{}/{}/{}",
19            r.owner, r.repo, r.branch, repo_rel
20        )
21    }
22    // fn try_entry(r: &RepoRef, out: &Path, repo_rel: &str) -> Option<PathBuf> {
23    //     let url = Self::raw_url(r, repo_rel);
24    //     if let Ok(bytes) = http_get(&url) {
25    //         let local = out.join(repo_rel);
26    //         if write_bytes(&local, &bytes).is_ok() { return Some(local); }
27    //     }
28    //     None
29    // }
30    // fn branch_candidates(branch: &str) -> Vec<String> {
31    //     if branch != "HEAD" { return vec![branch.to_string()]; }
32    //     vec!["main".into(), "master".into(), "trunk".into()]
33    // }
34    fn fetch_file(
35        r: &RepoRef,
36        out_root: &Path,
37        repo_rel: &str,
38    ) -> Result<PathBuf, Box<dyn std::error::Error>> {
39        let url = Self::raw_url(r, repo_rel);
40        let bytes = http_get(&url)?;
41        let local = out_root.join(repo_rel);
42        // Check for HTML/404 response
43        let is_html = bytes.starts_with(b"<!doctype html") || bytes.starts_with(b"<html");
44        if is_html {
45            let _ = std::fs::remove_file(&local);
46            return Err(format!("GitHub: Got HTML/404 for {}", url).into());
47        }
48        if let Some(p) = local.parent() {
49            fs::create_dir_all(p)?;
50        }
51        write_bytes(&local, &bytes)?;
52        Ok(local)
53    }
54}
55
56impl Provider for GitHub {
57    fn parse_url(&self, url: &str) -> Option<RepoRef> {
58        let u = Url::parse(url).ok()?;
59        if u.domain()? != "github.com" {
60            return None;
61        }
62        let segs = u.path_segments()?.collect::<Vec<_>>();
63        if segs.len() < 2 {
64            return None;
65        }
66
67        let owner = segs[0].to_string();
68        let repo = segs[1].trim_end_matches(".git").to_string();
69
70        if segs.len() == 2 {
71            return Some(RepoRef {
72                host: "github.com".into(),
73                owner,
74                repo,
75                branch: "HEAD".into(),
76                path: None,
77            });
78        }
79
80        // Accept GitHub views: blob/tree/raw/blame
81        match segs[2] {
82            "blob" | "tree" | "raw" | "blame" if segs.len() >= 4 => {
83                // <view>/<ref>/<path...>   where <ref> can be a branch name or a commit SHA
84                let branch = segs[3].to_string();
85                let path = if segs.len() > 4 {
86                    Some(segs[4..].join("/"))
87                } else {
88                    None
89                };
90                Some(RepoRef {
91                    host: "github.com".into(),
92                    owner,
93                    repo,
94                    branch,
95                    path,
96                })
97            }
98            _ => {
99                // Fallback: treat remainder as repo path with unknown ref
100                let path = if segs.len() > 2 {
101                    Some(segs[2..].join("/"))
102                } else {
103                    None
104                };
105                Some(RepoRef {
106                    host: "github.com".into(),
107                    owner,
108                    repo,
109                    branch: "HEAD".into(),
110                    path,
111                })
112            }
113        }
114    }
115    fn fetch_tal_tree(
116        &self,
117        r: &RepoRef,
118        out_root: &Path,
119    ) -> Result<FetchResult, Box<dyn std::error::Error>> {
120        let entry_rel = match &r.path {
121            Some(p)
122                if p.to_ascii_lowercase().ends_with(".tal")
123                    || p.to_ascii_lowercase().ends_with(".rom")
124                    || p.to_ascii_lowercase().ends_with(".rom.txt")
125                    || p.to_ascii_lowercase().ends_with(".orca")
126                    || p.to_ascii_lowercase().ends_with(".bas") =>
127            {
128                p.replace('\\', "/")
129            }
130            _ => return Err(
131                "github: URL must point to a .tal, .rom, .rom.txt, .orca, or .bas file; not guessing entries"
132                    .into(),
133            ),
134        };
135
136        // Check cache for entry file first
137        let entry_local = out_root.join(&entry_rel);
138        if entry_local.exists() {
139            eprintln!("[GitHub] Using cached file: {}", entry_local.display());
140            return Ok(FetchResult {
141                entry_local: entry_local.clone(),
142                all_files: vec![entry_local],
143            });
144        }
145        let entry_local = Self::fetch_file(r, out_root, &entry_rel)?;
146        let mut all = vec![entry_local.clone()];
147        // Recursively fetch includes for .tal files
148        if entry_rel.to_ascii_lowercase().ends_with(".tal") {
149            let mut visited: HashSet<String> = [entry_rel.clone()].into_iter().collect();
150            let mut q: VecDeque<(String, PathBuf)> =
151                VecDeque::from([(entry_rel.clone(), entry_local.clone())]);
152            let entry_dir = Path::new(&entry_rel)
153                .parent()
154                .map(|p| p.to_string_lossy().replace('\\', "/"))
155                .unwrap_or_default();
156            while let Some((curr_rel, curr_local)) = q.pop_front() {
157                let src = fs::read_to_string(&curr_local).unwrap_or_default();
158                for inc in parse_includes(&src) {
159                    let target = resolve_include(&curr_rel, &inc);
160                    if !visited.insert(target.clone()) {
161                        continue;
162                    }
163                    let mut attempts: Vec<(String, Option<std::path::PathBuf>)> = Vec::new();
164                    let mut success = None;
165                    let mut errors = vec![];
166                    // First attempt: entry_dir + target, unless target already starts with entry_dir
167                    let first_attempt = if !entry_dir.is_empty() && !target.starts_with(&entry_dir)
168                    {
169                        format!("{}/{}", entry_dir, target)
170                    } else {
171                        target.clone()
172                    };
173                    attempts.push((first_attempt.clone(), None));
174                    match Self::fetch_file(r, out_root, &first_attempt) {
175                        Ok(loc) => {
176                            success = Some((first_attempt.clone(), loc));
177                        }
178                        Err(e) => {
179                            errors.push((first_attempt.clone(), format!("{}", e)));
180                        }
181                    }
182                    // Fallback 1: deduped path
183                    if success.is_none() {
184                        let parts: Vec<&str> = target.split('/').collect();
185                        if parts.len() >= 3 && parts[0] == parts[1] {
186                            let deduped = parts[1..].join("/");
187                            attempts.push((deduped.clone(), None));
188                            match Self::fetch_file(r, out_root, &deduped) {
189                                Ok(loc) => {
190                                    success = Some((deduped.clone(), loc));
191                                }
192                                Err(e) => {
193                                    errors.push((deduped.clone(), format!("{}", e)));
194                                }
195                            }
196                        }
197                    }
198                    // Fallback 2: walk up ancestor directories
199                    if success.is_none() {
200                        if let Some(fname) = Path::new(&target).file_name() {
201                            let mut ancestor = Path::new(&target);
202                            while let Some(parent) = ancestor.parent() {
203                                let try_path = parent.join(fname);
204                                let try_str = try_path.to_string_lossy().replace('\\', "/");
205                                if try_str == target {
206                                    break;
207                                }
208                                attempts.push((try_str.clone(), None));
209                                match Self::fetch_file(r, out_root, &try_str) {
210                                    Ok(loc) => {
211                                        success = Some((try_str.clone(), loc));
212                                        break;
213                                    }
214                                    Err(e) => {
215                                        errors.push((try_str.clone(), format!("{}", e)));
216                                    }
217                                }
218                                ancestor = parent;
219                            }
220                        }
221                    }
222                    // Fallback 3: try just the filename in the entry file's directory
223                    if success.is_none() {
224                        if let Some(fname) = Path::new(&target).file_name() {
225                            if !entry_dir.is_empty() {
226                                let entry_dir_path = Path::new(&entry_dir).join(fname);
227                                let entry_dir_str =
228                                    entry_dir_path.to_string_lossy().replace('\\', "/");
229                                attempts.push((entry_dir_str.clone(), None));
230                                match Self::fetch_file(r, out_root, &entry_dir_str) {
231                                    Ok(loc) => {
232                                        success = Some((entry_dir_str.clone(), loc));
233                                    }
234                                    Err(e) => {
235                                        errors.push((entry_dir_str.clone(), format!("{}", e)));
236                                    }
237                                }
238                            }
239                        }
240                    }
241                    if let Some((path, loc)) = success {
242                        all.push(loc.clone());
243                        q.push_back((path, loc));
244                        continue;
245                    }
246                    // No fallback possible, return error with all attempts
247                    let error_msg = format!(
248                        "Failed to fetch include '{}'. Attempts: {}. Errors: {}",
249                        inc,
250                        attempts
251                            .iter()
252                            .map(|(p, _)| format!("'{}'", p))
253                            .collect::<Vec<_>>()
254                            .join(", "),
255                        errors
256                            .iter()
257                            .map(|(p, e)| format!("{}: {}", p, e))
258                            .collect::<Vec<_>>()
259                            .join("; ")
260                    );
261                    return Err(error_msg.into());
262                }
263            }
264        }
265        Ok(FetchResult {
266            entry_local: all[0].clone(),
267            all_files: all,
268        })
269
270        // for b in Self::branch_candidates(&r_in.branch) {
271        //     let r = RepoRef { branch: b.clone(), ..r_in.clone() };
272
273        //     let candidates: Vec<String> = match &r.path {
274        //         Some(p) if p.ends_with(".tal") => vec![p.clone()],
275        //         Some(p) if !p.is_empty()       => repo_entry_guesses().iter().map(|g| format!("{}/{}", p.trim_end_matches('/'), g)).collect(),
276        //         _ => repo_entry_guesses().iter().map(|s| s.to_string()).collect(),
277        //     };
278
279        //     if let Some((entry_rel, entry_local)) =
280        //         candidates.into_iter().find_map(|c| Self::try_entry(&r, out_root, &c).map(|l| (c, l)))
281        //     {
282        //         let mut visited = HashSet::new();
283        //         let mut all = vec![entry_local.clone()];
284        //         visited.insert(entry_rel.clone());
285        //         let mut q = VecDeque::new();
286        //         q.push_back((entry_rel, entry_local));
287
288        //         let mut steps = 0usize;
289        //         while let Some((repo_rel, local)) = q.pop_front() {
290        //             if steps >= 200 { break; }
291        //             steps += 1;
292
293        //             let src = fs::read_to_string(&local).unwrap_or_default();
294        //             for inc in parse_includes(&src) {
295        //                 let target = resolve_include(&repo_rel, &inc);
296        //                 if !visited.insert(target.clone()) { continue; }
297        //                 let url = Self::raw_url(&r, &target);
298        //                 println!("Fetching include from URL: {}", url);
299        //                 match http_get(&url) {
300        //                     Ok(bytes) => {
301        //                         let loc = out_root.join(&target);
302        //                         if write_bytes(&loc, &bytes).is_ok() {
303        //                             all.push(loc.clone());
304        //                             q.push_back((target, loc));
305        //                         }
306        //                     }
307        //                     Err(_) => eprintln!("warning: include not found in GitHub repo: {} (branch {})", target, r.branch),
308        //                 }
309        //             }
310        //         }
311        //         return Ok(FetchResult { entry_local: all[0].clone(), all_files: all });
312        //     }
313        // }
314        // Err("Could not find an entry .tal in GitHub repo across tried branches".into())
315    }
316
317    fn parse_git_url(&self, url: &str) -> Option<(RepoRef, String)> {
318        // Normalize double git@ to single git@
319        let normalized_url = if url.starts_with("git@git@") {
320            &url[4..] // Remove one "git@" prefix
321        } else {
322            url
323        };
324
325        // Handle .git/ format: git@github.com:owner/repo.git/path/file.tal
326        if let Some(git_pos) = normalized_url.find(".git/") {
327            let repo_part = &normalized_url[..git_pos + 4]; // Include ".git"
328            let file_part = &normalized_url[git_pos + 5..]; // Skip ".git/"
329
330            if repo_part.starts_with("git@github.com:") {
331                let path_part = &repo_part[15..repo_part.len() - 4]; // Remove "git@github.com:" and ".git"
332                let segments: Vec<&str> = path_part.split('/').collect();
333
334                if segments.len() >= 2 {
335                    let owner = segments[0].to_string();
336                    let repo = segments[1].to_string();
337
338                    let repo_ref = RepoRef {
339                        host: "github.com".to_string(),
340                        owner: owner.clone(),
341                        repo: repo.clone(),
342                        branch: "main".to_string(), // Default to main for .git format
343                        path: Some(file_part.to_string()),
344                    };
345
346                    let url_git = format!("git@github.com:{}/{}.git", owner, repo);
347                    return Some((repo_ref, url_git));
348                }
349            }
350        }
351
352        // Handle git@github.com:owner/repo/branch/path/file.tal (SSH format)
353        if let Some(path_part) = normalized_url.strip_prefix("git@github.com:") {
354            // Remove "git@github.com:"
355            let segments: Vec<&str> = path_part.split('/').collect();
356
357            if segments.len() < 4 {
358                return None;
359            }
360
361            let owner = segments[0].to_string();
362            let repo = segments[1].to_string();
363
364            // Skip "tree" segment if present
365            let (branch, path_start) = if segments[2] == "tree" {
366                (segments[3].to_string(), 4)
367            } else {
368                (segments[2].to_string(), 3)
369            };
370
371            let path = if segments.len() > path_start {
372                Some(segments[path_start..].join("/"))
373            } else {
374                None
375            };
376
377            let repo_ref = RepoRef {
378                host: "github.com".to_string(),
379                owner,
380                repo,
381                branch,
382                path,
383            };
384
385            let url_git = format!("git@github.com:{}/{}.git", repo_ref.owner, repo_ref.repo);
386            return Some((repo_ref, url_git));
387        }
388        // Handle git@https://... or git@http://... (strip git@ prefix and parse as HTTPS)
389        else if normalized_url.starts_with("git@https://")
390            || normalized_url.starts_with("git@http://")
391        {
392            let stripped_url = &normalized_url[4..]; // Remove "git@" prefix
393            if let Some(repo_ref) = self.parse_url(stripped_url) {
394                let url_git = format!(
395                    "https://github.com/{}/{}.git",
396                    repo_ref.owner, repo_ref.repo
397                );
398                return Some((repo_ref, url_git));
399            }
400        }
401
402        None
403    }
404}