Skip to main content

jouet_github/
lib.rs

1//! A small blocking GitHub client: finding a token that works, GraphQL, and
2//! the few REST reads GraphQL has no equivalent for.
3//!
4//! Tokens are tried in order: `JOUET_GITHUB_TOKEN`, `GH_TOKEN`,
5//! `GITHUB_TOKEN`, then `gh auth token`. There is no separate probe: the
6//! first real request checks the token, and a 401 moves on to the next
7//! source, so a stale `GH_TOKEN` costs one rejected request rather than a
8//! failed command. `JOUET_GITHUB_API_URL` points the client at another API
9//! root, such as a GitHub Enterprise server or a test's fake server.
10//!
11//! [`Client::put`] is the REST write for the merge and update-branch
12//! endpoints GraphQL has no safe equivalent of; [`Client::write`] sends any
13//! other method, for routines.
14//!
15//! [`Client::get`] sends `If-None-Match` when given an ETag, so an unchanged
16//! poll comes back as a 304 that costs nothing against the rate limit.
17
18pub mod entire;
19pub mod entire_forge;
20pub mod forge;
21pub mod forgejo;
22pub mod gitlab;
23pub mod host;
24pub mod pr;
25pub mod rest;
26pub mod tls;
27
28use serde::Deserialize;
29use serde_json::Value;
30use std::process::Command;
31use std::time::{Duration, SystemTime, UNIX_EPOCH};
32
33const ENV_SOURCES: [&str; 3] = ["JOUET_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN"];
34const GH_SOURCE: &str = "gh auth token";
35/// Retries after a 5xx, a dropped connection or a secondary rate limit.
36const RETRIES: u32 = 3;
37/// The longest a `Retry-After` header can make one retry wait.
38const MAX_RETRY_AFTER: u64 = 60;
39
40pub struct Client {
41    agent: ureq::Agent,
42    api_url: String,
43    /// Token sources not yet tried, in order.
44    sources: Vec<&'static str>,
45    tried: Vec<String>,
46    token: Option<Token>,
47    rejected: Vec<&'static str>,
48}
49
50struct Token {
51    source: &'static str,
52    value: String,
53}
54
55/// A REST response from [`Client::get`]. `status` is 304, with an empty
56/// body, when the ETag given still matches.
57#[derive(Debug)]
58pub struct Rest {
59    pub status: u16,
60    pub etag: Option<String>,
61    /// The `Link` header, which names the next page of a list.
62    pub link: Option<String>,
63    pub body: String,
64}
65
66/// Why a request got no usable response.
67#[derive(Debug)]
68pub enum Failure {
69    /// The primary rate limit is used up until `reset`, in Unix seconds.
70    RateLimited {
71        reset: i64,
72    },
73    Other(String),
74}
75
76impl Failure {
77    pub fn message(&self) -> String {
78        match self {
79            Self::RateLimited { reset } => format!(
80                "GitHub rate limit exhausted; it resets in {}",
81                jouet_output::time::age(reset - jouet_output::time::now())
82            ),
83            Self::Other(message) => message.clone(),
84        }
85    }
86}
87
88impl From<Failure> for String {
89    fn from(f: Failure) -> Self {
90        f.message()
91    }
92}
93
94/// What a request sends: a GraphQL POST, a REST write with its method, or
95/// (with neither) a GET.
96#[derive(Clone, Copy)]
97enum Body<'a> {
98    Post(&'a str),
99    Rest(Method, &'a str),
100}
101
102/// The REST methods that write.
103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
104pub enum Method {
105    Post,
106    Put,
107    Patch,
108    Delete,
109}
110
111impl Method {
112    pub fn parse(name: &str) -> Option<Self> {
113        match name.to_ascii_uppercase().as_str() {
114            "POST" => Some(Self::Post),
115            "PUT" => Some(Self::Put),
116            "PATCH" => Some(Self::Patch),
117            "DELETE" => Some(Self::Delete),
118            _ => None,
119        }
120    }
121}
122
123/// A response: its status, ETag, `Link` header and body.
124struct Sent {
125    status: u16,
126    etag: Option<String>,
127    link: Option<String>,
128    text: String,
129}
130
131#[derive(Deserialize)]
132struct GraphqlResponse {
133    data: Option<Value>,
134    #[serde(default)]
135    errors: Vec<GraphqlError>,
136}
137
138#[derive(Deserialize)]
139struct GraphqlError {
140    #[serde(rename = "type")]
141    kind: Option<String>,
142    message: String,
143}
144
145impl Client {
146    pub fn from_env() -> Self {
147        let api = std::env::var("JOUET_GITHUB_API_URL")
148            .ok()
149            .filter(|u| !u.is_empty())
150            .unwrap_or_else(|| "https://api.github.com".into());
151        let agent = ureq::Agent::config_builder()
152            .tls_config(crate::tls::config())
153            .http_status_as_error(false)
154            .timeout_global(Some(Duration::from_secs(30)))
155            .user_agent(concat!("jouet/", env!("CARGO_PKG_VERSION")))
156            .build()
157            .into();
158        Self {
159            agent,
160            api_url: api.trim_end_matches('/').to_string(),
161            sources: ENV_SOURCES.into_iter().chain([GH_SOURCE]).collect(),
162            tried: Vec::new(),
163            token: None,
164            rejected: Vec::new(),
165        }
166    }
167
168    /// The same client, sending to another API root, such as a test's
169    /// fake server.
170    pub fn with_api_url(mut self, url: &str) -> Self {
171        self.api_url = url.trim_end_matches('/').to_string();
172        self
173    }
174
175    /// The same client, using only `token`: no environment variable and no
176    /// `gh`. For a test's fake server.
177    pub fn with_token(mut self, token: &str) -> Self {
178        self.sources = Vec::new();
179        self.tried = Vec::new();
180        self.token = Some(Token {
181            source: "the given token",
182            value: token.to_string(),
183        });
184        self
185    }
186
187    /// Runs a GraphQL query and returns its `data`. Paths GitHub reports as
188    /// `NOT_FOUND` come back as `null` in the data; every other error fails
189    /// the call.
190    pub fn graphql(&mut self, query: &str, variables: &Value) -> Result<Value, String> {
191        self.graphql_with(query, variables, true)
192    }
193
194    /// Runs a GraphQL mutation. Unlike a query it is not retried after a 5xx
195    /// or a dropped connection, which can follow a write that succeeded.
196    pub fn mutate(&mut self, query: &str, variables: &Value) -> Result<Value, String> {
197        self.graphql_with(query, variables, false)
198    }
199
200    fn graphql_with(
201        &mut self,
202        query: &str,
203        variables: &Value,
204        idempotent: bool,
205    ) -> Result<Value, String> {
206        let body = serde_json::json!({ "query": query, "variables": variables }).to_string();
207        let url = format!("{}/graphql", self.api_url);
208        let Sent { status, text, .. } =
209            self.send(&url, Some(Body::Post(&body)), None, idempotent)?;
210        if status != 200 {
211            return Err(format!("GitHub returned {status}: {}", message(&text)));
212        }
213        let resp: GraphqlResponse = serde_json::from_str(&text)
214            .map_err(|e| format!("GitHub sent a response jouet could not read: {e}"))?;
215        if let Some(e) = resp
216            .errors
217            .iter()
218            .find(|e| e.kind.as_deref() != Some("NOT_FOUND"))
219        {
220            return Err(match e.kind.as_deref() {
221                Some("RATE_LIMITED") => format!("GitHub rate limit: {}", e.message),
222                _ => format!("GitHub: {}", e.message),
223            });
224        }
225        resp.data
226            .ok_or_else(|| "GitHub returned no data".to_string())
227    }
228
229    /// GETs a REST path such as `/repos/OWNER/NAME/activity`. A 404 comes
230    /// back as `None`.
231    pub fn rest_get(&mut self, path: &str) -> Result<Option<Value>, String> {
232        let url = format!("{}{path}", self.api_url);
233        let Sent { status, text, .. } = self.send(&url, None, None, true)?;
234        match status {
235            200 => serde_json::from_str(&text)
236                .map(Some)
237                .map_err(|e| format!("GitHub sent a response jouet could not read: {e}")),
238            404 => Ok(None),
239            _ => Err(format!("GitHub returned {status}: {}", message(&text))),
240        }
241    }
242
243    /// GETs a REST path such as `/repos/o/r/pulls/1`, or a full URL,
244    /// sending `If-None-Match` when `etag` is given. Statuses other than 401
245    /// are the caller's to read, and an exhausted rate limit says when it
246    /// resets, so a caller can wait it out.
247    pub fn get(&mut self, path: &str, etag: Option<&str>) -> Result<Rest, Failure> {
248        let url = if path.starts_with('/') {
249            format!("{}{path}", self.api_url)
250        } else {
251            path.to_string()
252        };
253        let sent = self.send(&url, None, etag, true)?;
254        Ok(sent.into())
255    }
256
257    /// PUTs a JSON body to a REST path such as `/repos/o/r/pulls/1/merge`.
258    /// A write is not retried after a 5xx or a dropped connection, which can
259    /// follow one that succeeded. Statuses other than 401 are the caller's to
260    /// read.
261    pub fn put(&mut self, path: &str, body: &Value) -> Result<Rest, Failure> {
262        self.write(Method::Put, path, body)
263    }
264
265    /// Sends a JSON body to a REST path with a method that writes. Like
266    /// [`Client::put`], it is not retried after a 5xx or a dropped
267    /// connection, and statuses other than 401 are the caller's to read.
268    pub fn write(&mut self, method: Method, path: &str, body: &Value) -> Result<Rest, Failure> {
269        let url = format!("{}{path}", self.api_url);
270        let body = body.to_string();
271        let sent = self.send(&url, Some(Body::Rest(method, &body)), None, false)?;
272        Ok(sent.into())
273    }
274
275    /// Sends a request with the first token GitHub accepts: `body` when
276    /// there is one, else a GET (conditional when `etag` is given).
277    fn send(
278        &mut self,
279        url: &str,
280        body: Option<Body<'_>>,
281        etag: Option<&str>,
282        idempotent: bool,
283    ) -> Result<Sent, Failure> {
284        loop {
285            let token = match self.token.take() {
286                Some(token) => token,
287                None => self.next_token().map_err(Failure::Other)?,
288            };
289            let sent = match self.request(&token.value, url, body, etag, idempotent) {
290                Ok(sent) => sent,
291                // The token was not rejected, so it is kept for the next
292                // request (such as the one after a rate limit resets).
293                Err(failure) => {
294                    self.token = Some(token);
295                    return Err(failure);
296                }
297            };
298            if sent.status == 401 {
299                self.rejected.push(token.source);
300                continue;
301            }
302            if !self.rejected.is_empty() {
303                let rejected = std::mem::take(&mut self.rejected);
304                let verb = if rejected.len() == 1 { "was" } else { "were" };
305                eprintln!(
306                    "jouet: {} {verb} rejected (401); using {}",
307                    join_and(&rejected),
308                    token.source
309                );
310            }
311            self.token = Some(token);
312            return Ok(sent);
313        }
314    }
315
316    fn next_token(&mut self) -> Result<Token, String> {
317        while !self.sources.is_empty() {
318            let source = self.sources.remove(0);
319            let value = if source == GH_SOURCE {
320                gh_auth_token()
321            } else {
322                std::env::var(source).ok()
323            };
324            let Some(value) = value
325                .map(|v| v.trim().to_string())
326                .filter(|v| !v.is_empty())
327            else {
328                continue;
329            };
330            // The same token under two names would only be rejected twice.
331            if self.tried.contains(&value) {
332                continue;
333            }
334            self.tried.push(value.clone());
335            return Ok(Token { source, value });
336        }
337        Err(if self.rejected.is_empty() {
338            "no GitHub token: set JOUET_GITHUB_TOKEN, GH_TOKEN or GITHUB_TOKEN, or run `gh auth login`"
339                .into()
340        } else {
341            format!(
342                "GitHub rejected every token (401): {}; run `gh auth login` or set JOUET_GITHUB_TOKEN",
343                join_and(&self.rejected)
344            )
345        })
346    }
347
348    /// Sends one request, retrying secondary rate limits with jittered
349    /// backoff, and when it is `idempotent` 5xx responses and connections
350    /// dropped before or during the response too.
351    fn request(
352        &self,
353        token: &str,
354        url: &str,
355        body: Option<Body<'_>>,
356        etag: Option<&str>,
357        idempotent: bool,
358    ) -> Result<Sent, Failure> {
359        let mut attempt = 0;
360        loop {
361            let auth = format!("bearer {token}");
362            let sent = match body {
363                Some(Body::Post(body)) => self
364                    .agent
365                    .post(url)
366                    .header("Authorization", auth)
367                    .content_type("application/json")
368                    .send(body),
369                Some(Body::Rest(method, body)) => {
370                    let req = match method {
371                        Method::Post => self.agent.post(url),
372                        Method::Put => self.agent.put(url),
373                        Method::Patch => self.agent.patch(url),
374                        Method::Delete => self.agent.delete(url).force_send_body(),
375                    };
376                    req.header("Authorization", auth)
377                        .header("Accept", "application/vnd.github+json")
378                        .header("X-GitHub-Api-Version", "2022-11-28")
379                        .content_type("application/json")
380                        .send(body)
381                }
382                None => {
383                    let mut req = self
384                        .agent
385                        .get(url)
386                        .header("Authorization", auth)
387                        .header("Accept", "application/vnd.github+json")
388                        .header("X-GitHub-Api-Version", "2022-11-28");
389                    if let Some(etag) = etag {
390                        req = req.header("If-None-Match", etag);
391                    }
392                    req.call()
393                }
394            }
395            .and_then(|mut resp| {
396                let text = resp
397                    .body_mut()
398                    .with_config()
399                    .limit(256 * 1024 * 1024)
400                    .read_to_string()?;
401                Ok((resp, text))
402            });
403            let (resp, text) = match sent {
404                Ok(sent) => sent,
405                Err(e) if idempotent && attempt < RETRIES => {
406                    eprintln!("jouet: request to GitHub failed ({e}); retrying");
407                    attempt += 1;
408                    std::thread::sleep(backoff(attempt));
409                    continue;
410                }
411                Err(e) => return Err(Failure::Other(format!("could not reach {url}: {e}"))),
412            };
413            let status = resp.status().as_u16();
414            let header = |name: &str| {
415                resp.headers()
416                    .get(name)
417                    .and_then(|v| v.to_str().ok())
418                    .map(str::to_string)
419            };
420            let retry_after = header("retry-after").and_then(|v| v.parse::<u64>().ok());
421            let remaining = header("x-ratelimit-remaining");
422            let reset = header("x-ratelimit-reset").and_then(|v| v.parse::<i64>().ok());
423            let etag = header("etag");
424            let link = header("link");
425            if status == 403 || status == 429 {
426                if remaining.as_deref() == Some("0") {
427                    let reset = reset.unwrap_or_else(|| jouet_output::time::now() + 60);
428                    return Err(Failure::RateLimited { reset });
429                }
430                let secondary = retry_after.is_some()
431                    || text.to_ascii_lowercase().contains("secondary rate limit");
432                if secondary && attempt < RETRIES {
433                    attempt += 1;
434                    let wait = retry_after.map_or_else(
435                        || backoff(attempt),
436                        |s| Duration::from_secs(s.min(MAX_RETRY_AFTER)),
437                    );
438                    eprintln!(
439                        "jouet: GitHub secondary rate limit; retrying in {}s",
440                        wait.as_secs()
441                    );
442                    std::thread::sleep(wait);
443                    continue;
444                }
445                if secondary {
446                    return Err(Failure::Other(format!(
447                        "GitHub secondary rate limit: {}",
448                        message(&text)
449                    )));
450                }
451            }
452            if status >= 500 && idempotent && attempt < RETRIES {
453                eprintln!("jouet: GitHub returned {status}; retrying");
454                attempt += 1;
455                std::thread::sleep(backoff(attempt));
456                continue;
457            }
458            return Ok(Sent {
459                status,
460                etag,
461                link,
462                text,
463            });
464        }
465    }
466}
467
468impl From<Sent> for Rest {
469    fn from(sent: Sent) -> Self {
470        Self {
471            status: sent.status,
472            etag: sent.etag,
473            link: sent.link,
474            body: sent.text,
475        }
476    }
477}
478
479/// The URL of the next page in a `Link` header, if it names one.
480pub fn next_page(link: &str) -> Option<String> {
481    link.split(',').find_map(|part| {
482        let (url, params) = part.split_once(';')?;
483        params
484            .split(';')
485            .any(|p| p.trim() == r#"rel="next""#)
486            .then(|| {
487                url.trim()
488                    .trim_start_matches('<')
489                    .trim_end_matches('>')
490                    .to_string()
491            })
492    })
493}
494
495/// Reads a token from `gh`, with `GH_TOKEN` and `GITHUB_TOKEN` removed so
496/// that gh reports its own stored token rather than echoing one that was
497/// already tried.
498fn gh_auth_token() -> Option<String> {
499    let out = Command::new("gh")
500        .args(["auth", "token"])
501        .env_remove("GH_TOKEN")
502        .env_remove("GITHUB_TOKEN")
503        .output()
504        .ok()?;
505    out.status
506        .success()
507        .then(|| String::from_utf8_lossy(&out.stdout).into_owned())
508}
509
510/// `500ms ยท 2^(attempt-1)` plus up to 250ms of jitter.
511pub(crate) fn backoff(attempt: u32) -> Duration {
512    let jitter = SystemTime::now()
513        .duration_since(UNIX_EPOCH)
514        .map_or(0, |d| d.subsec_millis() % 250);
515    Duration::from_millis(500 * 2u64.pow(attempt - 1) + u64::from(jitter))
516}
517
518/// The `message` of a GitHub error body, or the body itself.
519pub(crate) fn message(text: &str) -> String {
520    serde_json::from_str::<Value>(text)
521        .ok()
522        .and_then(|v| v.get("message")?.as_str().map(str::to_string))
523        .unwrap_or_else(|| text.trim().chars().take(200).collect())
524}
525
526pub(crate) fn join_and(items: &[&str]) -> String {
527    match items {
528        [] => String::new(),
529        [one] => (*one).to_string(),
530        [rest @ .., last] => format!("{} and {last}", rest.join(", ")),
531    }
532}
533
534#[cfg(test)]
535mod tests {
536    use super::*;
537
538    #[test]
539    fn lists_read_as_prose() {
540        assert_eq!(join_and(&["GH_TOKEN"]), "GH_TOKEN");
541        assert_eq!(
542            join_and(&["GH_TOKEN", "GITHUB_TOKEN", "gh auth token"]),
543            "GH_TOKEN, GITHUB_TOKEN and gh auth token"
544        );
545    }
546
547    #[test]
548    fn the_next_page_comes_from_the_link_header() {
549        let link = r#"<https://api.github.com/x?page=1>; rel="prev", <https://api.github.com/x?page=3>; rel="next""#;
550        assert_eq!(
551            next_page(link).as_deref(),
552            Some("https://api.github.com/x?page=3")
553        );
554        assert_eq!(
555            next_page(r#"<https://api.github.com/x?page=1>; rel="prev""#),
556            None
557        );
558    }
559
560    #[test]
561    fn error_messages_come_from_the_body() {
562        assert_eq!(
563            message(r#"{"message":"Bad credentials"}"#),
564            "Bad credentials"
565        );
566        assert_eq!(message("upstream timeout\n"), "upstream timeout");
567    }
568}