gh_file_curler/
lib.rs

1use reqwest::{blocking::Client, header::AUTHORIZATION};
2use std::{fs, path::Path};
3
4#[derive(Debug, Clone)]
5pub struct GhfcFile {
6    pub name: String,
7    pub content: Vec<u8>,
8}
9#[derive(Debug, Clone)]
10pub struct Files(pub Vec<GhfcFile>);
11
12/// If you want a `paths` entry to be the root, use `""` - might not work properly if you use "/"
13///
14/// `token`s can be generated in your Github settings
15pub fn fetch_dir(
16    user: &str,
17    repo: &str,
18    paths: &[&str],
19    recurse: bool,
20    token: &str,
21) -> Result<Files, String> {
22    _fetch_dir(user, repo, None, paths, recurse, token)
23}
24
25/// Identical to `fetch()`, except writes files immediately
26pub fn speedrun(
27    user: &str,
28    repo: &str,
29    out: &str,
30    paths: &[&str],
31    recurse: bool,
32    token: &str,
33) -> Result<Files, String> {
34    _fetch_dir(user, repo, Some(out), paths, recurse, token)
35}
36
37fn _fetch_dir(
38    user: &str,
39    repo: &str,
40    speedrun: Option<&str>,
41    paths: &[&str],
42    recurse: bool,
43    token: &str,
44) -> Result<Files, String> {
45    let client = Client::builder()
46        .user_agent("gh-file-curler")
47        .build()
48        .unwrap();
49    let mut out = Files(vec![]);
50    for path in paths {
51        let url = format!("https://api.github.com/repos/{user}/{repo}/contents{path}");
52        let json = client
53            .get(&url)
54            .header(AUTHORIZATION, format!("Bearer {token}"))
55            .send()
56            .unwrap()
57            .json::<serde_json::Value>()
58            .unwrap();
59        if json.as_array().is_none() {
60            return Err(format!("{json}"));
61        }
62        let json = json.as_array().unwrap();
63        for file in json {
64            if Some("file") == file["type"].as_str() {
65                if let Some(name) = file["name"].as_str() {
66                    if file["download_url"].as_str().is_some() {
67                        // println!("{path}/{name}");
68                        let f = fetch(user, repo, &[&format!("{path}/{name}")])
69                            .unwrap()
70                            .0[0]
71                            .clone();
72                        out.0.push(f.clone());
73                        if let Some(s) = speedrun {
74                            f.write_to(s);
75                        }
76                    }
77                }
78            } else if Some("dir") == file["type"].as_str() && recurse {
79                if let Some(name) = file["name"].as_str() {
80                    for x in _fetch_dir(
81                        user,
82                        repo,
83                        speedrun,
84                        &[&format!("{path}/{name}")],
85                        true,
86                        token,
87                    )
88                    .unwrap()
89                    .0
90                    {
91                        out.0.push(x);
92                    }
93                }
94            }
95        }
96    }
97    Ok(out)
98}
99
100pub fn fetch(user: &str, repo: &str, files: &[&str]) -> Result<Files, String> {
101    let client = Client::builder()
102        .user_agent("gh-file-curler")
103        .build()
104        .unwrap();
105    let mut out = Files(vec![]);
106    for file in files {
107        let url = format!("https://raw.githubusercontent.com/{user}/{repo}/main/{file}");
108        let mut content = client.get(&url).send().unwrap().bytes();
109        let mut i = 0;
110        while content.is_err() && i < 3 {
111            content = client.get(&url).send().unwrap().bytes();
112            i += 1;
113        }
114        if content.is_err() {
115            return Err(format!(
116                "multiple requests to {url} failed (e.g. timed out)"
117            ));
118        }
119        let content = content.unwrap();
120        let f = GhfcFile {
121            name: file.to_string(),
122            content: content.to_vec(),
123        };
124        out.0.push(f);
125    }
126    Ok(out)
127}
128
129impl Files {
130    pub fn write_to(self, path: &str) {
131        for f in self.0 {
132            f.write_to(path);
133        }
134    }
135}
136/// Most useful on a `fetch()` call for one file
137pub fn wrapped_first(f: Result<Files, String>) -> Result<Vec<u8>, String> {
138    if let Ok(f) = f {
139        let x = f.0[0].clone().content;
140        if x != b"404: Not Found" {
141            Ok(x)
142        } else {
143            Err(format!("could not find {}", f.0[0].name))
144        }
145    } else {
146        Err(f.unwrap_err())
147    }
148}
149impl GhfcFile {
150    pub fn write_to(self, path: &str) {
151        let p = format!("{path}/{}", self.name);
152        fs::create_dir_all(Path::new(&p).parent().unwrap()).unwrap();
153        fs::write(p, self.content).unwrap();
154    }
155}
gh_file_curler/lib.rs

gh_file_curler/
lib.rs
