Skip to main content

runnerd_core/github/
archives.rs

1//! What the action archives read from GitHub (`archives::Source`, ADR 0147): a repository's canonical name and
2//! visibility, its default branch's head, its tree and files, with the token through the API, and an action's archive,
3//! downloaded from codeload.github.com without it. An archive's download never carries the token, is never asked again
4//! with it, and follows no redirect: what a private repository holds cannot reach the store, whatever its visibility
5//! was when it was looked up.
6
7use std::os::unix::fs::OpenOptionsExt as _;
8use std::path::Path;
9use std::sync::atomic::{AtomicU64, Ordering};
10
11use reqwest::Method;
12use serde::Deserialize;
13use tokio::io::AsyncWriteExt as _;
14
15use super::{Error, Github, GithubApi as _};
16use crate::archives::{ActionRepository, Source, Tree, TreeFile};
17use crate::sync::lock;
18
19/// Where action archives are downloaded from, anonymously.
20pub(super) const CODELOAD: &str = "https://codeload.github.com";
21
22/// How long one archive's download may take, its redirect included.
23pub(super) const DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
24
25#[derive(Deserialize)]
26struct RepositoryBody {
27    full_name: String,
28    visibility: crate::api::Visibility,
29    default_branch: String,
30}
31
32#[derive(Deserialize)]
33struct RefBody {
34    object: RefObject,
35}
36
37#[derive(Deserialize)]
38struct RefObject {
39    sha: String,
40}
41
42#[derive(Deserialize)]
43struct TreeBody {
44    #[serde(default)]
45    truncated: bool,
46    tree: Vec<TreeEntry>,
47}
48
49#[derive(Deserialize)]
50struct TreeEntry {
51    path: String,
52    #[serde(rename = "type")]
53    kind: String,
54    sha: String,
55    #[serde(default)]
56    size: u64,
57}
58
59#[derive(Deserialize)]
60struct BlobBody {
61    content: String,
62    encoding: String,
63    size: u64,
64}
65
66/// A path segment as a URL carries it: a branch name may hold `#`, `%` or `?`.
67fn escaped(segment: &str) -> String {
68    segment
69        .bytes()
70        .map(|byte| {
71            if byte.is_ascii_alphanumeric() || matches!(byte, b'/' | b'-' | b'_' | b'.') {
72                char::from(byte).to_string()
73            } else {
74                format!("%{byte:02X}")
75            }
76        })
77        .collect()
78}
79
80/// A failed download, with what failed and never the address.
81fn download_error(path: &str, what: impl std::fmt::Display) -> Error {
82    Error::Download(format!("{path}: {what}"))
83}
84
85impl Source for Github {
86    async fn action_repository(&self, repository: &str) -> Result<ActionRepository, Error> {
87        let body: RepositoryBody = self.get(&format!("/repos/{repository}")).await?;
88        Ok(ActionRepository {
89            full_name: body.full_name,
90            public: body.visibility == crate::api::Visibility::Public,
91            default_branch: body.default_branch,
92        })
93    }
94
95    async fn branch_head(&self, repository: &str, branch: &str) -> Result<String, Error> {
96        // Kept for a conditional request: an unchanged head is a 304, which costs no rate limit.
97        let path = format!("/repos/{repository}/git/ref/heads/{}", escaped(branch));
98        let body: RefBody = self.get(&path).await?;
99        Ok(body.object.sha)
100    }
101
102    async fn tree(&self, repository: &str, commit: &str) -> Result<Tree, Error> {
103        let path = format!("/repos/{repository}/git/trees/{commit}?recursive=1");
104        let body: Result<TreeBody, Error> = self.get(&path).await;
105        // A commit's tree never changes, and a large one would crowd out the answers kept for polls.
106        lock(&self.etags).remove(&path);
107        let body = body?;
108        Ok(Tree {
109            truncated: body.truncated,
110            files: body
111                .tree
112                .into_iter()
113                .filter(|entry| entry.kind == "blob")
114                .map(|entry| TreeFile {
115                    path: entry.path,
116                    blob: entry.sha,
117                    size: entry.size,
118                })
119                .collect(),
120        })
121    }
122
123    async fn blob(&self, repository: &str, blob: &str, limit: u64) -> Result<Vec<u8>, Error> {
124        let path = format!("/repos/{repository}/git/blobs/{blob}");
125        let body: Result<BlobBody, Error> = self.get(&path).await;
126        lock(&self.etags).remove(&path);
127        let body = body?;
128        if body.size > limit {
129            return Err(download_error(
130                &path,
131                format_args!("larger than {limit} bytes"),
132            ));
133        }
134        if body.encoding != "base64" {
135            return Err(download_error(&path, "not base64"));
136        }
137        base64(&body.content).ok_or_else(|| download_error(&path, "not base64"))
138    }
139
140    async fn tarball(
141        &self,
142        repository: &str,
143        commit: &str,
144        to: &Path,
145        limit: u64,
146        received: &AtomicU64,
147    ) -> Result<u64, Error> {
148        // Anonymous, whatever GitHub answers: only what anyone may download is ever kept.
149        let path = format!("/{repository}/tar.gz/{commit}");
150        let response = self
151            .downloads
152            .get(format!("{}{path}", self.codeload))
153            .send()
154            .await
155            .map_err(|source| Error::Http {
156                method: Method::GET,
157                path: path.clone(),
158                source: source.without_url(),
159            })?;
160        let status = response.status();
161        if !status.is_success() {
162            return Err(Error::Status {
163                method: Method::GET,
164                path,
165                status,
166                message: "the archive was not sent".to_owned(),
167            });
168        }
169        if response
170            .content_length()
171            .is_some_and(|length| length > limit)
172        {
173            return Err(download_error(
174                &path,
175                format_args!("larger than {limit} bytes"),
176            ));
177        }
178        write_capped(response, &path, to, limit, received).await
179    }
180
181    fn spare_requests(&self) -> Option<u64> {
182        let rate_limit = self.spend().rate_limit?;
183        (rate_limit.reset > std::time::SystemTime::now()).then_some(rate_limit.remaining)
184    }
185}
186
187/// Streams `response`'s body into a new file `to`, readable by its owner only until it is published, failing once it
188/// passes `limit` bytes. Every chunk received counts in `received`, the one that passes `limit` too.
189async fn write_capped(
190    mut response: reqwest::Response,
191    path: &str,
192    to: &Path,
193    limit: u64,
194    received: &AtomicU64,
195) -> Result<u64, Error> {
196    let io = |error: std::io::Error| download_error(path, error);
197    let file = std::fs::OpenOptions::new()
198        .write(true)
199        .create_new(true)
200        .mode(0o600)
201        .open(to)
202        .map_err(io)?;
203    let mut file = tokio::fs::File::from_std(file);
204    let mut written: u64 = 0;
205    loop {
206        let chunk = response.chunk().await.map_err(|source| Error::Http {
207            method: Method::GET,
208            path: path.to_owned(),
209            source: source.without_url(),
210        })?;
211        let Some(chunk) = chunk else { break };
212        received.fetch_add(chunk.len() as u64, Ordering::Relaxed);
213        written += chunk.len() as u64;
214        if written > limit {
215            return Err(download_error(
216                path,
217                format_args!("larger than {limit} bytes"),
218            ));
219        }
220        file.write_all(&chunk).await.map_err(io)?;
221    }
222    file.flush().await.map_err(io)?;
223    Ok(written)
224}
225
226/// Standard base64 with padding, as GitHub's blobs are, ignoring line breaks; `None` if it is not.
227fn base64(text: &str) -> Option<Vec<u8>> {
228    let value = |byte: u8| -> Option<u32> {
229        Some(u32::from(match byte {
230            b'A'..=b'Z' => byte - b'A',
231            b'a'..=b'z' => byte - b'a' + 26,
232            b'0'..=b'9' => byte - b'0' + 52,
233            b'+' => 62,
234            b'/' => 63,
235            _ => return None,
236        }))
237    };
238    let clean: Vec<u8> = text
239        .bytes()
240        .filter(|byte| !byte.is_ascii_whitespace())
241        .collect();
242    if !clean.len().is_multiple_of(4) {
243        return None;
244    }
245    let mut out = Vec::with_capacity(clean.len() / 4 * 3);
246    for quad in clean.chunks(4) {
247        let pad = quad.iter().rev().take_while(|byte| **byte == b'=').count();
248        if pad > 2 {
249            return None;
250        }
251        let mut bits: u32 = 0;
252        for byte in &quad[..4 - pad] {
253            bits = bits << 6 | value(*byte)?;
254        }
255        bits <<= if pad == 2 {
256            12
257        } else {
258            6 * u32::from(pad == 1)
259        };
260        let bytes = bits.to_be_bytes();
261        out.extend_from_slice(&bytes[1..4 - pad]);
262    }
263    Some(out)
264}
265
266#[cfg(test)]
267mod tests {
268    use super::base64;
269
270    #[test]
271    fn base64_decodes_what_github_sends_with_its_line_breaks() {
272        assert_eq!(
273            base64("dXNlczog\nYUBi\n").as_deref(),
274            Some(&b"uses: a@b"[..])
275        );
276        assert_eq!(base64("YQ==").as_deref(), Some(&b"a"[..]));
277        assert_eq!(base64("YWI=").as_deref(), Some(&b"ab"[..]));
278        assert_eq!(base64("YQ="), None);
279        assert_eq!(base64("Y!==",), None);
280    }
281}