git_mirror/provider/
gitlab.rs

1/*
2 * Copyright (c) 2017-2018 Pascal Bach
3 *
4 * SPDX-License-Identifier:     MIT
5 */
6
7// Used for error and debug logging
8use log::{debug, error, trace, warn};
9
10use reqwest::StatusCode;
11use reqwest::blocking::Client;
12use reqwest::header::{HeaderMap, HeaderValue};
13
14use crate::provider::{Desc, Mirror, MirrorError, MirrorResult, Provider};
15
16#[derive(Debug)]
17pub struct GitLab {
18    pub url: String,
19    pub group: String,
20    pub use_http: bool,
21    pub private_token: Option<String>,
22    pub recursive: bool,
23}
24
25/// A project from the GitLab API
26#[derive(Deserialize, Debug, Clone)]
27struct Project {
28    description: String,
29    web_url: String,
30    ssh_url_to_repo: String,
31    http_url_to_repo: String,
32}
33
34/// A (sub)group from the GitLab API
35#[derive(Deserialize, Debug, Clone)]
36struct Group {
37    id: u64,
38}
39
40// Number of items per page to request
41const PER_PAGE: u8 = 100;
42
43impl GitLab {
44    fn get_paged<T: serde::de::DeserializeOwned>(
45        &self,
46        url: &str,
47        client: &Client,
48        headers: &HeaderMap,
49    ) -> Result<Vec<T>, String> {
50        let mut results: Vec<T> = Vec::new();
51
52        for page in 1..u32::MAX {
53            let url = format!("{url}?per_page={PER_PAGE}&page={page}");
54            trace!("URL: {url}");
55
56            let res = client
57                .get(&url)
58                .headers(headers.clone())
59                .send()
60                .map_err(|e| format!("Unable to connect to: {url} ({e})"))?;
61
62            debug!("HTTP Status Received: {}", res.status());
63
64            if res.status() != StatusCode::OK {
65                if res.status() == StatusCode::UNAUTHORIZED {
66                    return Err(format!(
67                        "API call received unautorized ({}) for: {}. \
68                         Please make sure the `GITLAB_PRIVATE_TOKEN` environment \
69                         variable is set.",
70                        res.status(),
71                        url
72                    ));
73                } else {
74                    return Err(format!(
75                        "API call received invalid status ({}) for : {}",
76                        res.status(),
77                        url
78                    ));
79                }
80            }
81
82            let has_next = match res.headers().get("x-next-page") {
83                None => {
84                    trace!("No more pages, x-next-page header missing.");
85                    false
86                }
87                Some(n) => {
88                    if n.is_empty() {
89                        trace!("No more pages, x-next-page-header empty.");
90                        false
91                    } else {
92                        trace!("Next page: {n:?}");
93                        true
94                    }
95                }
96            };
97
98            let results_page: Vec<T> = serde_json::from_reader(res)
99                .map_err(|e| format!("Unable to parse response as JSON ({e})"))?;
100
101            results.extend(results_page);
102
103            if !has_next {
104                break;
105            }
106        }
107        Ok(results)
108    }
109
110    fn get_projects(
111        &self,
112        id: &str,
113        client: &Client,
114        headers: &HeaderMap,
115    ) -> Result<Vec<Project>, String> {
116        let url = format!("{}/api/v4/groups/{}/projects", self.url, id);
117
118        self.get_paged::<Project>(&url, client, headers)
119    }
120
121    fn get_subgroups(
122        &self,
123        id: &str,
124        client: &Client,
125        headers: &HeaderMap,
126    ) -> Result<Vec<String>, String> {
127        let url = format!("{}/api/v4/groups/{}/subgroups", self.url, id);
128
129        let groups = self.get_paged::<Group>(&url, client, headers)?;
130
131        let mut subgroups: Vec<String> = vec![id.to_owned()];
132
133        for group in groups {
134            subgroups.extend(self.get_subgroups(&format!("{}", group.id), client, headers)?);
135        }
136
137        Ok(subgroups)
138    }
139}
140
141impl Provider for GitLab {
142    fn get_label(&self) -> String {
143        format!("{}/{}", self.url, self.group)
144    }
145
146    fn get_mirror_repos(&self) -> Result<Vec<MirrorResult>, String> {
147        let client = Client::new();
148
149        let use_http = self.use_http;
150
151        let mut headers = HeaderMap::new();
152        if let Some(ref token) = self.private_token {
153            match HeaderValue::from_str(token) {
154                Ok(token) => {
155                    headers.insert("PRIVATE-TOKEN", token);
156                }
157                Err(err) => {
158                    error!("Unable to parse PRIVATE_TOKEN: {err}");
159                }
160            }
161        } else {
162            warn!("PRIVATE_TOKEN not set")
163        }
164
165        let groups = if self.recursive {
166            self.get_subgroups(&self.group, &client, &headers).or_else(
167                |e| -> Result<Vec<String>, String> {
168                    warn!("Unable to get subgroups: {e}");
169                    Ok(vec![self.group.clone()])
170                },
171            )?
172        } else {
173            vec![self.group.clone()]
174        };
175
176        let mut projects: Vec<Project> = Vec::new();
177
178        for group in groups {
179            projects.extend(self.get_projects(&group, &client, &headers)?);
180        }
181
182        let mut mirrors: Vec<MirrorResult> = Vec::new();
183
184        for p in projects {
185            match serde_yaml::from_str::<Desc>(&p.description) {
186                Ok(desc) => {
187                    if desc.skip {
188                        mirrors.push(Err(MirrorError::Skip(p.web_url)));
189                        continue;
190                    }
191                    trace!("{0} -> {1}", desc.origin, p.ssh_url_to_repo);
192                    let destination = if use_http {
193                        p.http_url_to_repo
194                    } else {
195                        p.ssh_url_to_repo
196                    };
197                    let m = Mirror {
198                        origin: desc.origin,
199                        destination,
200                        refspec: desc.refspec,
201                        lfs: desc.lfs,
202                    };
203                    mirrors.push(Ok(m));
204                }
205                Err(e) => {
206                    mirrors.push(Err(MirrorError::Description(p.web_url, e)));
207                }
208            }
209        }
210
211        Ok(mirrors)
212    }
213}
git_mirror/provider/gitlab.rs

git_mirror/provider/
gitlab.rs
