Skip to main content

jouet_github/
rest.rs

1//! A blocking REST client for GitLab and Forgejo, shaped like [`Client`]:
2//! tokens tried in order, a 401 moving on to the next source, retries after
3//! a 5xx or a dropped connection, and pages followed through the `Link`
4//! header.
5//!
6//! A host's tokens come from the environment variables that belong to it
7//! ([`forge::token_vars`](crate::forge): GitLab's to `GITLAB_HOST` or
8//! gitlab.com, `FORGEJO_TOKEN` to codeberg.org, `GITEA_TOKEN` to gitea.com,
9//! a configured host's `token_env` to that host), then from glab's or tea's
10//! `config.yml`, which keep a token per host. No token is ever sent to a
11//! host it does not belong to. With no token at all a request goes without
12//! one, which public projects answer. Messages name a token's source, never
13//! its value.
14//!
15//! [`Client`]: crate::Client
16
17use crate::forge::{self, Forge};
18use crate::{Failure, Method, backoff, join_and, message};
19use serde_json::Value;
20use std::path::PathBuf;
21use std::time::Duration;
22
23/// Retries after a 5xx, a dropped connection or a 429.
24const RETRIES: u32 = 3;
25/// The longest a `Retry-After` header can make one retry wait.
26const MAX_RETRY_AFTER: u64 = 60;
27/// Pages of a list read at most.
28pub const MAX_PAGES: usize = 30;
29
30pub struct Rest {
31    agent: ureq::Agent,
32    forge: Forge,
33    host: String,
34    api_url: String,
35    /// Token sources not yet tried, in order.
36    sources: Vec<Source>,
37    tried: Vec<String>,
38    token: Option<Token>,
39    rejected: Vec<String>,
40}
41
42#[derive(Debug, Clone, PartialEq, Eq)]
43enum Source {
44    Env(String),
45    /// glab's or tea's `config.yml`.
46    Cli,
47    /// Every source used up: requests go without a token.
48    Anonymous,
49}
50
51struct Token {
52    source: Source,
53    /// `None` sends the request without a token.
54    value: Option<String>,
55}
56
57/// A REST response. `status` is 304, with an empty body, when the ETag
58/// given still matches.
59#[derive(Debug)]
60pub struct Response {
61    pub status: u16,
62    pub etag: Option<String>,
63    pub link: Option<String>,
64    pub body: String,
65}
66
67impl Rest {
68    pub fn new(forge: Forge, host: &str) -> Self {
69        let agent = ureq::Agent::config_builder()
70            .tls_config(crate::tls::config())
71            .http_status_as_error(false)
72            .timeout_global(Some(Duration::from_secs(30)))
73            .user_agent(concat!("jouet/", env!("CARGO_PKG_VERSION")))
74            .build()
75            .into();
76        let sources = forge::token_vars(forge, host)
77            .into_iter()
78            .map(Source::Env)
79            .chain([Source::Cli])
80            .collect();
81        Self {
82            agent,
83            forge,
84            host: forge::normalize(host),
85            api_url: forge::api_url(forge, host),
86            sources,
87            tried: Vec::new(),
88            token: None,
89            rejected: Vec::new(),
90        }
91    }
92
93    pub fn host(&self) -> &str {
94        &self.host
95    }
96
97    /// The web server's root: the API root without its `/api/v4` or
98    /// `/api/v1`, else `https://HOST`.
99    pub fn web_root(&self) -> String {
100        ["/api/v4", "/api/v1"]
101            .iter()
102            .find_map(|suffix| self.api_url.strip_suffix(suffix))
103            .map_or_else(|| format!("https://{}", self.host), str::to_string)
104    }
105
106    /// GETs a path under the API root, or a full URL (a next page),
107    /// sending `If-None-Match` when `etag` is given. Statuses other than 401
108    /// are the caller's to read.
109    pub fn get(&mut self, path: &str, etag: Option<&str>) -> Result<Response, Failure> {
110        let url = self.url(path);
111        self.send(&url, None, etag, true)
112    }
113
114    /// GETs JSON; a 404 is `None`, and any other status but 200 an error.
115    pub fn json(&mut self, path: &str) -> Result<Option<Value>, Failure> {
116        let resp = self.get(path, None)?;
117        self.read(path, resp)
118    }
119
120    /// Every page of a JSON list, up to [`MAX_PAGES`]; a 404 is `None`.
121    pub fn list(&mut self, path: &str) -> Result<Option<Vec<Value>>, Failure> {
122        let mut out = Vec::new();
123        let mut next = Some(path.to_string());
124        let mut pages = 0;
125        while let Some(path) = next.take() {
126            let resp = self.get(&path, None)?;
127            let link = resp.link.clone();
128            let Some(page) = self.read(&path, resp)? else {
129                return Ok(None);
130            };
131            out.extend(page.as_array().cloned().unwrap_or_default());
132            pages += 1;
133            if pages < MAX_PAGES {
134                next = link.as_deref().and_then(crate::next_page);
135            }
136        }
137        Ok(Some(out))
138    }
139
140    /// Sends a JSON body with a method that writes. It is not retried after
141    /// a 5xx or a dropped connection, which can follow a write that
142    /// succeeded.
143    pub fn write(&mut self, method: Method, path: &str, body: &Value) -> Result<Response, Failure> {
144        let url = self.url(path);
145        let body = body.to_string();
146        self.send(&url, Some((method, &body)), None, false)
147    }
148
149    /// Reads a response as JSON: 404 is `None`, other statuses but 2xx are
150    /// errors carrying the forge's message.
151    pub fn read(&self, path: &str, resp: Response) -> Result<Option<Value>, Failure> {
152        match resp.status {
153            200..=299 => serde_json::from_str(&resp.body).map(Some).map_err(|e| {
154                Failure::Other(format!(
155                    "{} sent {path} in a form jouet could not read: {e}",
156                    self.forge.name()
157                ))
158            }),
159            404 => Ok(None),
160            status => Err(self.refused(status, &resp.body)),
161        }
162    }
163
164    /// A failure for an unexpected status, with the forge's message.
165    pub fn refused(&self, status: u16, body: &str) -> Failure {
166        Failure::Other(format!(
167            "{} returned {status}: {}",
168            self.forge.name(),
169            error_message(body)
170        ))
171    }
172
173    fn url(&self, path: &str) -> String {
174        if path.starts_with('/') {
175            format!("{}{path}", self.api_url)
176        } else {
177            path.to_string()
178        }
179    }
180
181    fn send(
182        &mut self,
183        url: &str,
184        body: Option<(Method, &str)>,
185        etag: Option<&str>,
186        idempotent: bool,
187    ) -> Result<Response, Failure> {
188        loop {
189            let token = match self.token.take() {
190                Some(token) => token,
191                None => self.next_token(),
192            };
193            let sent = match self.request(token.value.as_deref(), url, body, etag, idempotent) {
194                Ok(sent) => sent,
195                Err(failure) => {
196                    self.token = Some(token);
197                    return Err(failure);
198                }
199            };
200            if sent.status == 401 {
201                if token.value.is_none() {
202                    return Err(Failure::Other(self.no_token()));
203                }
204                self.rejected.push(self.source_name(&token.source));
205                continue;
206            }
207            if !self.rejected.is_empty() && token.value.is_some() {
208                let rejected = std::mem::take(&mut self.rejected);
209                let names: Vec<&str> = rejected.iter().map(String::as_str).collect();
210                let verb = if names.len() == 1 { "was" } else { "were" };
211                eprintln!(
212                    "jouet: {} {verb} rejected by {} (401); using {}",
213                    join_and(&names),
214                    self.host,
215                    self.source_name(&token.source)
216                );
217            }
218            self.token = Some(token);
219            return Ok(sent);
220        }
221    }
222
223    /// The next token to try; once every source is used up, none, so a
224    /// public project can still be read.
225    fn next_token(&mut self) -> Token {
226        while !self.sources.is_empty() {
227            let source = self.sources.remove(0);
228            let value = match &source {
229                Source::Env(var) => std::env::var(var).ok(),
230                Source::Cli => cli_token(self.forge, &self.host),
231                Source::Anonymous => None,
232            };
233            let Some(value) = value
234                .map(|v| v.trim().to_string())
235                .filter(|v| !v.is_empty())
236            else {
237                continue;
238            };
239            if self.tried.contains(&value) {
240                continue;
241            }
242            self.tried.push(value.clone());
243            return Token {
244                source,
245                value: Some(value),
246            };
247        }
248        Token {
249            source: Source::Anonymous,
250            value: None,
251        }
252    }
253
254    fn source_name(&self, source: &Source) -> String {
255        match (source, self.forge) {
256            (Source::Env(var), _) => var.to_string(),
257            (Source::Cli, Forge::GitLab) => "glab's config".into(),
258            (Source::Cli, _) => "tea's config".into(),
259            (Source::Anonymous, _) => "no token".into(),
260        }
261    }
262
263    /// Why a request that needed a token got none, naming what was tried.
264    fn no_token(&self) -> String {
265        let vars = forge::token_vars(self.forge, &self.host);
266        let vars = if vars.is_empty() {
267            format!(
268                "the variable named by token_env in [forges.\"{}\"]",
269                self.host
270            )
271        } else {
272            join_or(&vars.iter().map(String::as_str).collect::<Vec<_>>())
273        };
274        let cli = match self.forge {
275            Forge::GitLab => "run `glab auth login`",
276            _ => "run `tea login add`",
277        };
278        if self.rejected.is_empty() {
279            format!("{} needs a token: set {vars}, or {cli}", self.host)
280        } else {
281            let names: Vec<&str> = self.rejected.iter().map(String::as_str).collect();
282            format!(
283                "{} rejected every token (401): {}; set {vars}, or {cli}",
284                self.host,
285                join_and(&names)
286            )
287        }
288    }
289
290    fn request(
291        &self,
292        token: Option<&str>,
293        url: &str,
294        body: Option<(Method, &str)>,
295        etag: Option<&str>,
296        idempotent: bool,
297    ) -> Result<Response, Failure> {
298        let auth = token.map(|t| match self.forge {
299            Forge::Forgejo => format!("token {t}"),
300            _ => format!("Bearer {t}"),
301        });
302        let mut attempt = 0;
303        loop {
304            let sent = match body {
305                Some((method, body)) => {
306                    let req = match method {
307                        Method::Post => self.agent.post(url),
308                        Method::Put => self.agent.put(url),
309                        Method::Patch => self.agent.patch(url),
310                        Method::Delete => self.agent.delete(url).force_send_body(),
311                    };
312                    let req = match &auth {
313                        Some(auth) => req.header("Authorization", auth),
314                        None => req,
315                    };
316                    req.header("Accept", "application/json")
317                        .content_type("application/json")
318                        .send(body)
319                }
320                None => {
321                    let mut req = self.agent.get(url).header("Accept", "application/json");
322                    if let Some(auth) = &auth {
323                        req = req.header("Authorization", auth);
324                    }
325                    if let Some(etag) = etag {
326                        req = req.header("If-None-Match", etag);
327                    }
328                    req.call()
329                }
330            }
331            .and_then(|mut resp| {
332                let text = resp
333                    .body_mut()
334                    .with_config()
335                    .limit(256 * 1024 * 1024)
336                    .read_to_string()?;
337                Ok((resp, text))
338            });
339            let name = self.forge.name();
340            let (resp, text) = match sent {
341                Ok(sent) => sent,
342                Err(e) if idempotent && attempt < RETRIES => {
343                    eprintln!("jouet: request to {name} failed ({e}); retrying");
344                    attempt += 1;
345                    std::thread::sleep(backoff(attempt));
346                    continue;
347                }
348                Err(e) => return Err(Failure::Other(format!("could not reach {url}: {e}"))),
349            };
350            let status = resp.status().as_u16();
351            let header = |name: &str| {
352                resp.headers()
353                    .get(name)
354                    .and_then(|v| v.to_str().ok())
355                    .map(str::to_string)
356            };
357            if status == 429 {
358                let retry_after = header("retry-after").and_then(|v| v.parse::<u64>().ok());
359                if attempt < RETRIES {
360                    attempt += 1;
361                    let wait = retry_after.map_or_else(
362                        || backoff(attempt),
363                        |s| Duration::from_secs(s.min(MAX_RETRY_AFTER)),
364                    );
365                    eprintln!("jouet: {name} rate limit; retrying in {}s", wait.as_secs());
366                    std::thread::sleep(wait);
367                    continue;
368                }
369                return Err(Failure::Other(format!(
370                    "{name} rate limit: {}",
371                    error_message(&text)
372                )));
373            }
374            if status >= 500 && idempotent && attempt < RETRIES {
375                eprintln!("jouet: {name} returned {status}; retrying");
376                attempt += 1;
377                std::thread::sleep(backoff(attempt));
378                continue;
379            }
380            return Ok(Response {
381                status,
382                etag: header("etag"),
383                link: header("link"),
384                body: text,
385            });
386        }
387    }
388}
389
390/// The `message` (GitLab, Forgejo) or `error` of an error body, or the
391/// start of the body.
392fn error_message(body: &str) -> String {
393    let parsed = serde_json::from_str::<Value>(body).ok();
394    let field = parsed.as_ref().and_then(|v| {
395        ["message", "error"].iter().find_map(|k| match &v[*k] {
396            Value::String(s) => Some(s.clone()),
397            Value::Null => None,
398            other => Some(other.to_string()),
399        })
400    });
401    field.unwrap_or_else(|| message(body))
402}
403
404/// A timestamp sent with the server's offset (`+01:00`), in UTC as GitHub
405/// sends them.
406pub fn utc(v: &Value) -> Option<String> {
407    v.as_str()
408        .and_then(jouet_output::time::parse_rfc3339)
409        .map(jouet_output::time::format_utc)
410}
411
412/// Percent-encodes everything but unreserved characters, for a path segment
413/// such as GitLab's `group%2Fproject` or a query value.
414pub fn encode(s: &str) -> String {
415    let mut out = String::new();
416    for b in s.bytes() {
417        if b.is_ascii_alphanumeric() || b"-_.~".contains(&b) {
418            out.push(char::from(b));
419        } else {
420            out.push_str(&format!("%{b:02X}"));
421        }
422    }
423    out
424}
425
426fn join_or(items: &[&str]) -> String {
427    match items {
428        [] => String::new(),
429        [one] => (*one).to_string(),
430        [rest @ .., last] => format!("{} or {last}", rest.join(", ")),
431    }
432}
433
434/// Reads a CLI's config for the token of a host.
435type ReadToken = fn(&str, &str) -> Option<String>;
436
437/// The token glab (for GitLab) or tea (for Forgejo) keeps for `host`.
438fn cli_token(forge: Forge, host: &str) -> Option<String> {
439    let (dir, read): (&str, ReadToken) = match forge {
440        Forge::GitLab => ("glab-cli", glab_token),
441        Forge::Forgejo => ("tea", tea_token),
442        Forge::GitHub | Forge::Entire => return None,
443    };
444    let override_dir = match forge {
445        Forge::GitLab => std::env::var_os("GLAB_CONFIG_DIR").map(PathBuf::from),
446        _ => None,
447    };
448    let mut dirs: Vec<PathBuf> = override_dir.into_iter().collect();
449    if let Some(xdg) = std::env::var_os("XDG_CONFIG_HOME").map(PathBuf::from) {
450        dirs.push(xdg.join(dir));
451    }
452    if let Some(home) = std::env::home_dir() {
453        dirs.push(home.join(".config").join(dir));
454        dirs.push(home.join("Library/Application Support").join(dir));
455    }
456    dirs.iter()
457        .filter_map(|d| std::fs::read_to_string(d.join("config.yml")).ok())
458        .find_map(|text| read(&text, host))
459}
460
461/// A YAML scalar without its quotes.
462fn unquote(value: &str) -> String {
463    let v = value.trim();
464    let v = v
465        .strip_prefix('"')
466        .and_then(|v| v.strip_suffix('"'))
467        .or_else(|| v.strip_prefix('\'').and_then(|v| v.strip_suffix('\'')))
468        .unwrap_or(v);
469    v.to_string()
470}
471
472/// A `key: value` line, or `key:` with an empty value.
473fn key_value(line: &str) -> Option<(String, String)> {
474    let line = line.trim();
475    if let Some(key) = line.strip_suffix(':') {
476        return Some((unquote(key), String::new()));
477    }
478    let (key, value) = line.split_once(": ")?;
479    Some((unquote(key), unquote(value)))
480}
481
482/// `hosts.<host>.token` in glab's config.
483fn glab_token(text: &str, host: &str) -> Option<String> {
484    let mut in_hosts = false;
485    let mut host_indent: Option<usize> = None;
486    let mut current: Option<String> = None;
487    for line in text.lines() {
488        let trimmed = line.trim_start();
489        if trimmed.is_empty() || trimmed.starts_with('#') {
490            continue;
491        }
492        let indent = line.len() - trimmed.len();
493        if indent == 0 {
494            in_hosts = trimmed.trim_end() == "hosts:";
495            host_indent = None;
496            current = None;
497            continue;
498        }
499        let Some((key, value)) = key_value(trimmed).filter(|_| in_hosts) else {
500            continue;
501        };
502        if host_indent.is_none_or(|h| indent <= h) {
503            host_indent = Some(indent);
504            current = Some(forge::normalize(&key));
505        } else if key == "token"
506            && current.as_deref() == Some(host)
507            && !value.is_empty()
508            && !value.starts_with("!!null")
509        {
510            return Some(value);
511        }
512    }
513    None
514}
515
516/// The token of the login in tea's config whose `url` is on `host`.
517fn tea_token(text: &str, host: &str) -> Option<String> {
518    let mut in_logins = false;
519    let mut logins: Vec<(Option<String>, Option<String>)> = Vec::new();
520    for line in text.lines() {
521        let trimmed = line.trim_start();
522        if trimmed.is_empty() || trimmed.starts_with('#') {
523            continue;
524        }
525        if line.len() == trimmed.len() {
526            in_logins = trimmed.trim_end() == "logins:";
527            continue;
528        }
529        if !in_logins {
530            continue;
531        }
532        let entry = match trimmed.strip_prefix("- ") {
533            Some(rest) => {
534                logins.push((None, None));
535                rest
536            }
537            None => trimmed,
538        };
539        let (Some(login), Some((key, value))) = (logins.last_mut(), key_value(entry)) else {
540            continue;
541        };
542        match key.as_str() {
543            "url" => login.0 = Some(value),
544            "token" => login.1 = Some(value),
545            _ => {}
546        }
547    }
548    logins.into_iter().find_map(|(url, token)| {
549        let url = url?;
550        let rest = url.split_once("://").map_or(url.as_str(), |(_, r)| r);
551        let login_host = rest.split('/').next()?;
552        (forge::normalize(login_host) == host)
553            .then_some(token)
554            .flatten()
555            .filter(|t| !t.is_empty())
556    })
557}
558
559#[cfg(test)]
560mod tests {
561    use super::*;
562
563    #[test]
564    fn glab_keeps_a_token_per_host() {
565        let config = "git_protocol: ssh\nhosts:\n    gitlab.com:\n        api_protocol: https\n        token: glpat-one\n    git.example.com:\n        token: 'glpat-two'\n";
566        assert_eq!(
567            glab_token(config, "gitlab.com").as_deref(),
568            Some("glpat-one")
569        );
570        assert_eq!(
571            glab_token(config, "git.example.com").as_deref(),
572            Some("glpat-two")
573        );
574        assert_eq!(glab_token(config, "other.example.com"), None);
575        // A token kept in the keyring leaves the field empty.
576        assert_eq!(
577            glab_token("hosts:\n  gitlab.com:\n    token:\n", "gitlab.com"),
578            None
579        );
580    }
581
582    #[test]
583    fn tea_matches_a_login_by_its_url() {
584        let config = "logins:\n    - name: work\n      url: https://git.example.com\n      token: one\n    - name: codeberg\n      url: https://codeberg.org\n      token: \"two\"\n      default: true\npreferences:\n    editor: false\n";
585        assert_eq!(tea_token(config, "codeberg.org").as_deref(), Some("two"));
586        assert_eq!(tea_token(config, "git.example.com").as_deref(), Some("one"));
587        assert_eq!(tea_token(config, "gitea.com"), None);
588    }
589
590    #[test]
591    fn error_bodies_name_the_forges_message() {
592        assert_eq!(
593            error_message(r#"{"message":"401 Unauthorized"}"#),
594            "401 Unauthorized"
595        );
596        assert_eq!(
597            error_message(r#"{"message":{"base":["is behind"]}}"#),
598            r#"{"base":["is behind"]}"#
599        );
600        assert_eq!(
601            error_message(r#"{"error":"insufficient_scope"}"#),
602            "insufficient_scope"
603        );
604    }
605}