gr/
io.rs

1use crate::{
2    api_defaults,
3    cmds::{
4        merge_request::MergeRequestResponse,
5        project::{Member, Project},
6    },
7    http::{self, Headers, Request},
8    log_info,
9    remote::RemoteURL,
10    time::{self, Seconds},
11    Result,
12};
13use regex::Regex;
14use serde::Serialize;
15use std::{
16    ffi::OsStr,
17    fmt::{self, Display, Formatter},
18    rc::Rc,
19};
20
21/// A trait that handles the execution of processes with a finite lifetime. For
22/// example, it can be an in-memory process for testing or a shell command doing
23/// I/O. It handles all processes that do not conform with the HTTP protocol.
24/// For that, check the `HttpRunner`
25pub trait TaskRunner {
26    type Response;
27    fn run<T>(&self, cmd: T) -> Result<Self::Response>
28    where
29        T: IntoIterator,
30        T::Item: AsRef<OsStr>;
31}
32
33/// A trait for the HTTP protocol. Implementers need to conform with the HTTP
34/// constraints and requirements. Implementers accept a `Request` that wraps
35/// headers, payloads and HTTP methods. Clients can potentially do HTTP calls
36/// against a remote server or mock the responses for testing purposes.
37pub trait HttpRunner {
38    type Response;
39    fn run<T: Serialize>(&self, cmd: &mut Request<T>) -> Result<Self::Response>;
40    /// Return the number of API MAX PAGES allowed for the given Request.
41    fn api_max_pages<T: Serialize>(&self, cmd: &Request<T>) -> u32;
42}
43
44type Title = String;
45type Description = String;
46
47#[derive(Clone, Debug)]
48pub enum CmdInfo {
49    StatusModified(bool),
50    RemoteUrl(RemoteURL),
51    Branch(String),
52    CommitSummary(String),
53    CommitMessage(String),
54    CommitBody(Title, Description),
55    Project(Project),
56    Members(Vec<Member>),
57    MergeRequest(MergeRequestResponse),
58    MergeRequestsList(Vec<MergeRequestResponse>),
59    OutgoingCommits(String),
60    Ignore,
61    Exit,
62}
63
64#[derive(Clone, Debug, Builder)]
65pub struct ShellResponse {
66    #[builder(default)]
67    pub status: i32,
68    #[builder(default)]
69    pub body: String,
70}
71
72impl ShellResponse {
73    pub fn builder() -> ShellResponseBuilder {
74        ShellResponseBuilder::default()
75    }
76}
77
78/// Adapts lower level I/O HTTP/Shell outputs to a common Response.
79#[derive(Clone, Debug, Builder)]
80pub struct HttpResponse {
81    #[builder(default)]
82    pub status: i32,
83    #[builder(default)]
84    pub body: String,
85    /// Optional headers. Mostly used by HTTP downstream HTTP responses
86    #[builder(setter(into, strip_option), default)]
87    pub headers: Option<Headers>,
88    #[builder(setter(into), default)]
89    pub flow_control_headers: FlowControlHeaders,
90    #[builder(setter(into), default)]
91    pub local_cache: bool,
92}
93
94impl HttpResponse {
95    pub fn builder() -> HttpResponseBuilder {
96        HttpResponseBuilder::default()
97    }
98}
99
100#[derive(Clone, Debug, PartialEq)]
101pub enum ResponseField {
102    Body,
103    Status,
104    Headers,
105}
106
107impl HttpResponse {
108    pub fn header(&self, key: &str) -> Option<&str> {
109        self.headers
110            .as_ref()
111            .and_then(|h| h.get(key))
112            .map(|s| s.as_str())
113    }
114
115    pub fn get_page_headers(&self) -> Rc<Option<PageHeader>> {
116        self.flow_control_headers.get_page_header()
117    }
118
119    pub fn get_ratelimit_headers(&self) -> Rc<Option<RateLimitHeader>> {
120        self.flow_control_headers.get_rate_limit_header()
121    }
122
123    pub fn get_flow_control_headers(&self) -> &FlowControlHeaders {
124        &self.flow_control_headers
125    }
126
127    pub fn get_etag(&self) -> Option<&str> {
128        self.header("etag")
129    }
130
131    pub fn is_ok(&self, method: &http::Method) -> bool {
132        match method {
133            http::Method::HEAD => self.status == 200,
134            http::Method::GET => self.status == 200,
135            http::Method::POST => {
136                self.status >= 200 && self.status < 300 || self.status == 409 || self.status == 422
137            }
138            http::Method::PATCH | http::Method::PUT => self.status >= 200 && self.status < 300,
139        }
140    }
141
142    pub fn update_rate_limit_headers(&mut self, headers: RateLimitHeader) {
143        self.flow_control_headers.rate_limit_header = Rc::new(Some(headers));
144    }
145}
146
147const NEXT: &str = "next";
148const LAST: &str = "last";
149pub const LINK_HEADER: &str = "link";
150
151fn parse_link_headers(link: &str) -> PageHeader {
152    lazy_static! {
153        static ref RE_URL: Regex = Regex::new(r#"<([^>]+)>;\s*rel="([^"]+)""#).unwrap();
154        static ref RE_PAGE_NUMBER: Regex = Regex::new(r"[^(per_)]page=(\d+)").unwrap();
155        static ref RE_PER_PAGE: Regex = Regex::new(r"per_page=(\d+)").unwrap();
156    }
157    let mut page_header = PageHeader::new();
158    'links: for cap in RE_URL.captures_iter(link) {
159        if cap.len() > 2 && &cap[2] == NEXT {
160            // Capture per_page in next page if available to avoid re-computing
161            // this section in next matches like `first` and `last`
162            if let Some(per_page) = RE_PER_PAGE.captures(&cap[1]) {
163                if per_page.len() > 1 {
164                    let per_page = per_page[1].to_string();
165                    let per_page: u32 = per_page.parse().unwrap_or(api_defaults::DEFAULT_PER_PAGE);
166                    page_header.per_page = per_page;
167                }
168            } else {
169                page_header.per_page = api_defaults::DEFAULT_PER_PAGE;
170            };
171            let url = cap[1].to_string();
172            if let Some(page_cap) = RE_PAGE_NUMBER.captures(&url) {
173                if page_cap.len() == 2 {
174                    let page_number = page_cap[1].to_string();
175                    let page_number: u32 = page_number.parse().unwrap_or(0);
176                    let page = Page::new(&url, page_number);
177                    page_header.set_next_page(page);
178                    continue 'links;
179                }
180            }
181        }
182        // TODO pull code out - return a page and its type next or last.
183        if cap.len() > 2 && &cap[2] == LAST {
184            let url = cap[1].to_string();
185            if let Some(page_cap) = RE_PAGE_NUMBER.captures(&url) {
186                if page_cap.len() == 2 {
187                    let page_number = page_cap[1].to_string();
188                    let page_number: u32 = page_number.parse().unwrap_or(0);
189                    let page = Page::new(&url, page_number);
190                    page_header.set_last_page(page);
191                }
192            }
193        }
194    }
195    if page_header.per_page == 0 {
196        page_header.per_page = api_defaults::DEFAULT_PER_PAGE;
197    }
198    page_header
199}
200
201#[derive(Clone, Debug, Default)]
202pub struct PageHeader {
203    pub next: Option<Page>,
204    pub last: Option<Page>,
205    pub per_page: u32,
206}
207
208impl PageHeader {
209    pub fn new() -> Self {
210        Self::default()
211    }
212    pub fn set_next_page(&mut self, page: Page) {
213        self.next = Some(page);
214    }
215
216    pub fn set_last_page(&mut self, page: Page) {
217        self.last = Some(page);
218    }
219
220    pub fn next_page(&self) -> Option<&Page> {
221        self.next.as_ref()
222    }
223
224    pub fn last_page(&self) -> Option<&Page> {
225        self.last.as_ref()
226    }
227}
228
229pub fn parse_page_headers(headers: Option<&Headers>) -> Option<PageHeader> {
230    if let Some(headers) = headers {
231        let link = headers.get(LINK_HEADER)?;
232        return Some(parse_link_headers(link));
233    }
234    None
235}
236
237#[derive(Clone, Debug, PartialEq)]
238pub struct Page {
239    pub url: String,
240    pub number: u32,
241}
242
243impl Page {
244    pub fn new(url: &str, number: u32) -> Self {
245        Page {
246            url: url.to_string(),
247            number,
248        }
249    }
250
251    pub fn url(&self) -> &str {
252        &self.url
253    }
254}
255
256// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#exceeding-the-rate-limit
257
258pub const GITHUB_RATELIMIT_REMAINING: &str = "x-ratelimit-remaining";
259pub const GITHUB_RATELIMIT_RESET: &str = "x-ratelimit-reset";
260
261// Time to wait before retrying the next request - standard common header
262// Gitlab Docs: Retry-After
263pub const RETRY_AFTER: &str = "retry-after";
264
265// https://docs.gitlab.com/ee/administration/settings/user_and_ip_rate_limits.html
266
267// Internal processing is all in lowercase
268// Docs: RateLimit-Remaining
269pub const GITLAB_RATELIMIT_REMAINING: &str = "ratelimit-remaining";
270// Docs: RateLimit-Reset
271pub const GITLAB_RATELIMIT_RESET: &str = "ratelimit-reset";
272
273/// Unifies the different ratelimit headers available from the different remotes.
274/// Github API ratelimit headers:
275/// remaining: x-ratelimit-remaining
276/// reset: x-ratelimit-reset
277/// Gitlab API ratelimit headers:
278/// remaining: RateLimit-Remaining
279/// reset: RateLimit-Reset
280#[derive(Clone, Copy, Debug, Default)]
281pub struct RateLimitHeader {
282    // The number of requests remaining in the current rate limit window.
283    pub remaining: u32,
284    // Unix time-formatted time when the request quota is reset.
285    pub reset: Seconds,
286    // Time to wait before retrying the next request
287    pub retry_after: Seconds,
288}
289
290impl RateLimitHeader {
291    pub fn new(remaining: u32, reset: Seconds, retry_after: Seconds) -> Self {
292        RateLimitHeader {
293            remaining,
294            reset,
295            retry_after,
296        }
297    }
298}
299
300// Defaults:
301// https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
302// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-authenticated-users
303
304// Github 5000 requests per hour for authenticated users
305// Gitlab 2000 requests per minute for authenticated users
306// Most limiting Github 5000/60 = 83.33 requests per minute
307
308pub fn parse_ratelimit_headers(headers: Option<&Headers>) -> Option<RateLimitHeader> {
309    let mut ratelimit_header = RateLimitHeader::default();
310
311    // process remote headers and patch the defaults accordingly
312    if let Some(headers) = headers {
313        if let Some(retry_after) = headers.get(RETRY_AFTER) {
314            ratelimit_header.retry_after = Seconds::new(retry_after.parse::<u64>().unwrap_or(0));
315        }
316        if let Some(github_remaining) = headers.get(GITHUB_RATELIMIT_REMAINING) {
317            ratelimit_header.remaining = github_remaining.parse::<u32>().unwrap_or(0);
318            if let Some(github_reset) = headers.get(GITHUB_RATELIMIT_RESET) {
319                ratelimit_header.reset = Seconds::new(github_reset.parse::<u64>().unwrap_or(0));
320            }
321            log_info!("Header {}", ratelimit_header);
322            return Some(ratelimit_header);
323        }
324        if let Some(gitlab_remaining) = headers.get(GITLAB_RATELIMIT_REMAINING) {
325            ratelimit_header.remaining = gitlab_remaining.parse::<u32>().unwrap_or(0);
326            if let Some(gitlab_reset) = headers.get(GITLAB_RATELIMIT_RESET) {
327                ratelimit_header.reset = Seconds::new(gitlab_reset.parse::<u64>().unwrap_or(0));
328            }
329            log_info!("Header {}", ratelimit_header);
330            return Some(ratelimit_header);
331        }
332    }
333    None
334}
335
336impl Display for RateLimitHeader {
337    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
338        let reset = time::epoch_to_minutes_relative(self.reset);
339        write!(
340            f,
341            "RateLimitHeader: remaining: {}, reset in: {} minutes",
342            self.remaining, reset
343        )
344    }
345}
346
347#[derive(Clone, Debug, Default)]
348pub struct FlowControlHeaders {
349    page_header: Rc<Option<PageHeader>>,
350    rate_limit_header: Rc<Option<RateLimitHeader>>,
351}
352
353impl FlowControlHeaders {
354    pub fn new(
355        page_header: Rc<Option<PageHeader>>,
356        rate_limit_header: Rc<Option<RateLimitHeader>>,
357    ) -> Self {
358        FlowControlHeaders {
359            page_header,
360            rate_limit_header,
361        }
362    }
363
364    pub fn get_page_header(&self) -> Rc<Option<PageHeader>> {
365        self.page_header.clone()
366    }
367
368    pub fn get_rate_limit_header(&self) -> Rc<Option<RateLimitHeader>> {
369        self.rate_limit_header.clone()
370    }
371}
372
373#[cfg(test)]
374mod test {
375    use super::*;
376
377    #[test]
378    fn test_get_rate_limit_headers_github() {
379        let body = "responsebody";
380        let mut headers = Headers::new();
381        headers.set("x-ratelimit-remaining".to_string(), "30".to_string());
382        headers.set("x-ratelimit-reset".to_string(), "1658602270".to_string());
383        headers.set("retry-after".to_string(), "60".to_string());
384        let rate_limit_header = parse_ratelimit_headers(Some(&headers)).unwrap();
385        let flow_control_headers =
386            FlowControlHeaders::new(Rc::new(None), Rc::new(Some(rate_limit_header)));
387        let response = HttpResponse::builder()
388            .body(body.to_string())
389            .headers(headers)
390            .flow_control_headers(flow_control_headers)
391            .build()
392            .unwrap();
393        let ratelimit_headers = response.get_ratelimit_headers().unwrap();
394        assert_eq!(30, ratelimit_headers.remaining.clone());
395        assert_eq!(Seconds::new(1658602270), ratelimit_headers.reset);
396        assert_eq!(Seconds::new(60), ratelimit_headers.retry_after);
397    }
398
399    #[test]
400    fn test_get_rate_limit_headers_gitlab() {
401        let body = "responsebody";
402        let mut headers = Headers::new();
403        headers.set("ratelimit-remaining".to_string(), "30".to_string());
404        headers.set("ratelimit-reset".to_string(), "1658602270".to_string());
405        headers.set("retry-after".to_string(), "60".to_string());
406        let rate_limit_header = parse_ratelimit_headers(Some(&headers)).unwrap();
407        let flow_control_headers =
408            FlowControlHeaders::new(Rc::new(None), Rc::new(Some(rate_limit_header)));
409        let response = HttpResponse::builder()
410            .body(body.to_string())
411            .headers(headers)
412            .flow_control_headers(flow_control_headers)
413            .build()
414            .unwrap();
415        let ratelimit_headers = response.get_ratelimit_headers().unwrap();
416        assert_eq!(30, ratelimit_headers.remaining);
417        assert_eq!(Seconds::new(1658602270), ratelimit_headers.reset);
418        assert_eq!(Seconds::new(60), ratelimit_headers.retry_after);
419    }
420
421    #[test]
422    fn test_get_rate_limit_headers_camelcase_gitlab() {
423        let body = "responsebody";
424        let mut headers = Headers::new();
425        headers.set("RateLimit-remaining".to_string(), "30".to_string());
426        headers.set("rateLimit-reset".to_string(), "1658602270".to_string());
427        headers.set("Retry-After".to_string(), "60".to_string());
428        let rate_limit_header = parse_ratelimit_headers(Some(&headers));
429        let flow_control_headers =
430            FlowControlHeaders::new(Rc::new(None), Rc::new(rate_limit_header));
431        let response = HttpResponse::builder()
432            .body(body.to_string())
433            .headers(headers)
434            .flow_control_headers(flow_control_headers)
435            .build()
436            .unwrap();
437        let ratelimit_headers = response.get_ratelimit_headers();
438        assert!(ratelimit_headers.is_none());
439    }
440
441    #[test]
442    fn test_link_header_has_next_and_last_page() {
443        let link = r#"<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=2>; rel="next", <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last""#;
444        let page_headers = parse_link_headers(link);
445        assert_eq!(
446            "https://api.github.com/search/code?q=addClass+user%3Amozilla&page=2",
447            page_headers.next.as_ref().unwrap().url
448        );
449        assert_eq!(2, page_headers.next.unwrap().number);
450        assert_eq!(
451            "https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34",
452            page_headers.last.as_ref().unwrap().url
453        );
454        assert_eq!(34, page_headers.last.unwrap().number);
455    }
456
457    #[test]
458    fn test_link_header_has_no_next_page() {
459        let link = r#"<http://gitlab-web/api/v4/projects/tooling%2Fcli/members/all?id=tooling%2Fcli&page=1&per_page=20>; rel="first", <http://gitlab-web/api/v4/projects/tooling%2Fcli/members/all?id=tooling%2Fcli&page=1&per_page=20>; rel="last""#;
460        let page_headers = parse_link_headers(link);
461        assert_eq!(None, page_headers.next);
462    }
463
464    #[test]
465    fn test_link_header_has_first_next_and_last() {
466        let link = r#"<https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=2&per_page=20&sort=desc>; rel="next", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=1&per_page=20&sort=desc>; rel="first", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=91&per_page=20&sort=desc>; rel="last""#;
467        let page_headers = parse_link_headers(link);
468        assert_eq!(91, page_headers.last.unwrap().number);
469        assert_eq!(2, page_headers.next.unwrap().number);
470    }
471
472    #[test]
473    fn test_response_ok_status_get_request_200() {
474        assert!(HttpResponse::builder()
475            .status(200)
476            .build()
477            .unwrap()
478            .is_ok(&http::Method::GET));
479    }
480
481    #[test]
482    fn test_response_not_ok_if_get_request_400s() {
483        let not_ok_status = 400..=499;
484        for status in not_ok_status {
485            let response = HttpResponse::builder().status(status).build().unwrap();
486            assert!(!response.is_ok(&http::Method::GET));
487        }
488    }
489
490    #[test]
491    fn test_response_ok_status_post_request_201() {
492        assert!(HttpResponse::builder()
493            .status(201)
494            .build()
495            .unwrap()
496            .is_ok(&http::Method::POST));
497    }
498
499    #[test]
500    fn test_response_ok_if_post_request_409_422() {
501        // special case handled by the caller (merge_request)
502        let not_ok_status = [409, 422];
503        for status in not_ok_status.iter() {
504            let response = HttpResponse::builder().status(*status).build().unwrap();
505            assert!(response.is_ok(&http::Method::POST));
506        }
507    }
508
509    #[test]
510    fn test_response_not_ok_if_500s_any_case() {
511        let methods = [
512            http::Method::GET,
513            http::Method::POST,
514            http::Method::PATCH,
515            http::Method::PUT,
516        ];
517        let not_ok_status = 500..=599;
518        for status in not_ok_status {
519            for method in methods.iter() {
520                let response = HttpResponse::builder().status(status).build().unwrap();
521                assert!(!response.is_ok(method));
522            }
523        }
524    }
525
526    #[test]
527    fn test_link_headers_get_per_page_multiple_pages() {
528        let link = r#"<https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=2&per_page=20&sort=desc>; rel="next", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=1&per_page=20&sort=desc>; rel="first", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=91&per_page=20&sort=desc>; rel="last""#;
529        let page_headers = parse_link_headers(link);
530        assert_eq!(91, page_headers.last.unwrap().number);
531        assert_eq!(2, page_headers.next.unwrap().number);
532        assert_eq!(20, page_headers.per_page);
533    }
534
535    #[test]
536    fn test_link_headers_get_per_page_not_available_use_default() {
537        let link = r#"<https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=2&sort=desc>; rel="next", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=1&sort=desc>; rel="first", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=91&sort=desc>; rel="last""#;
538        let page_headers = parse_link_headers(link);
539        assert_eq!(91, page_headers.last.unwrap().number);
540        assert_eq!(2, page_headers.next.unwrap().number);
541        assert_eq!(api_defaults::DEFAULT_PER_PAGE, page_headers.per_page);
542    }
543
544    #[test]
545    fn test_link_headers_get_per_page_with_no_next_use_default() {
546        let link = r#"<https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=1&sort=desc>; rel="first", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=91&sort=desc>; rel="last""#;
547        let page_headers = parse_link_headers(link);
548        assert_eq!(91, page_headers.last.unwrap().number);
549        assert_eq!(None, page_headers.next);
550        assert_eq!(api_defaults::DEFAULT_PER_PAGE, page_headers.per_page);
551    }
552
553    #[test]
554    fn test_link_headers_get_per_page_available_in_last_only_use_default() {
555        let link = r#"<https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&page=2&sort=desc>; rel="next", <https://gitlab-web/api/v4/projects/15/pipelines?id=15&order_by=id&per_page=20&page=91&sort=desc>; rel="last""#;
556        let page_headers = parse_link_headers(link);
557        assert_eq!(91, page_headers.last.unwrap().number);
558        assert_eq!(2, page_headers.next.unwrap().number);
559        assert_eq!(api_defaults::DEFAULT_PER_PAGE, page_headers.per_page);
560    }
561}
gr/io.rs

gr/
io.rs
