comindly_core/
hosting.rs

1//! Lists the repositories of a GitHub organisation or user, or a GitLab group (with subgroups), so
2//! they can be picked and cloned. Works with github.com, GitHub Enterprise, gitlab.com and self-hosted GitLab.
3
4use anyhow::{bail, Context, Result};
5use serde::Deserialize;
6
7use crate::worktree::{Auth, NeedsToken};
8
9pub use comindly_api::hosting::{Provider, RemoteRepo, Source};
10
11/// Read `https://github.com/acme`, `gitlab.com/acme/platform` or `https://git.example.com/group`.
12pub fn parse(url: &str) -> Result<Source> {
13    let url = url.trim().trim_end_matches('/');
14    let clone_url = url.ends_with(".git") || url.starts_with("git@") || url.starts_with("ssh://");
15    let rest = url
16        .trim_start_matches("https://")
17        .trim_start_matches("http://")
18        .trim_start_matches("ssh://")
19        .trim_start_matches("git@");
20    let rest = rest.split(['?', '#']).next().unwrap_or_default();
21    let (host, path) = rest
22        .split_once(['/', ':'])
23        .context("give the organisation, group or repository, e.g. github.com/acme")?;
24    let path = path.trim_end_matches(".git").trim_matches('/');
25    let path = path.split("/-/").next().unwrap_or(path);
26    anyhow::ensure!(
27        !path.is_empty(),
28        "give the organisation, group or repository after the host"
29    );
30    let host = host.to_lowercase();
31    let provider = if host == "github.com" || host.contains("github") {
32        Provider::GitHub
33    } else {
34        Provider::GitLab
35    };
36    let parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
37    let (owner, repo) = match provider {
38        // github.com/orgs/acme/repositories is the organisation; github.com/acme/orders a repository.
39        Provider::GitHub if parts.first() == Some(&"orgs") => {
40            (parts.get(1).unwrap_or(&"").to_string(), None)
41        }
42        Provider::GitHub => (parts[0].to_string(), parts.get(1).map(|r| r.to_string())),
43        Provider::GitLab if clone_url && parts.len() >= 2 => {
44            let (group, name) = path.rsplit_once('/').unwrap_or_default();
45            (group.to_string(), Some(name.to_string()))
46        }
47        Provider::GitLab => (path.to_string(), None),
48    };
49    anyhow::ensure!(
50        !owner.is_empty(),
51        "give the organisation, group or repository after the host"
52    );
53    Ok(Source {
54        provider,
55        host,
56        owner,
57        repo,
58    })
59}
60
61/// Every repository of `source`, newest activity first.
62pub async fn list(source: &Source, auth: Option<&Auth>) -> Result<Vec<RemoteRepo>> {
63    let client = reqwest::Client::builder()
64        .user_agent(concat!("comindly/", env!("CARGO_PKG_VERSION")))
65        .build()?;
66    let mut repos = match source.provider {
67        Provider::GitHub => match &source.repo {
68            Some(name) => {
69                let api = github_api(&source.host);
70                let url = format!("{api}/repos/{}/{name}", source.owner);
71                let one = one(&client, &url, auth, false, &source.host)
72                    .await?
73                    .with_context(|| {
74                        format!("no repository {}/{name} on {}", source.owner, source.host)
75                    })?;
76                github_repo(&one).into_iter().collect()
77            }
78            None => github(&client, source, auth).await?,
79        },
80        Provider::GitLab => {
81            // `a/b` is a project or a subgroup; ask for the project first.
82            let path = match &source.repo {
83                Some(name) => format!("{}/{name}", source.owner),
84                None => source.owner.clone(),
85            };
86            let url = format!(
87                "https://{}/api/v4/projects/{}",
88                source.host,
89                path.replace('/', "%2F")
90            );
91            let project = if path.contains('/') {
92                one(&client, &url, auth, true, &source.host).await?
93            } else {
94                None
95            };
96            match project {
97                Some(p) => gitlab_repo(&p).into_iter().collect(),
98                None if source.repo.is_some() => bail!("no repository {path} on {}", source.host),
99                None => gitlab(&client, source, auth).await?,
100            }
101        }
102    };
103    repos.sort_by(|a, b| b.last_activity.cmp(&a.last_activity));
104    Ok(repos)
105}
106
107/// One object, or `None` when it is not found.
108async fn one(
109    client: &reqwest::Client,
110    url: &str,
111    auth: Option<&Auth>,
112    gitlab: bool,
113    host: &str,
114) -> Result<Option<serde_json::Value>> {
115    let mut req = client.get(url);
116    if let Some(a) = auth {
117        req = if gitlab {
118            req.header("PRIVATE-TOKEN", &a.token)
119        } else {
120            req.bearer_auth(&a.token)
121        };
122    }
123    let resp = req.send().await.with_context(|| format!("reach {host}"))?;
124    match resp.status().as_u16() {
125        404 => Ok(None),
126        401 | 403 => Err(NeedsToken {
127            host: host.to_string(),
128            username: crate::worktree::token_user(host).into(),
129        }
130        .into()),
131        s if !resp.status().is_success() => bail!("{host} answered {s}"),
132        _ => Ok(Some(resp.json().await?)),
133    }
134}
135
136fn github_api(host: &str) -> String {
137    if host == "github.com" {
138        "https://api.github.com".to_string()
139    } else {
140        format!("https://{host}/api/v3")
141    }
142}
143
144/// Fetch every page of a listing; `None` when the first page is not found.
145async fn pages(
146    client: &reqwest::Client,
147    url: &str,
148    auth: Option<&Auth>,
149    gitlab: bool,
150    host: &str,
151) -> Result<Option<Vec<serde_json::Value>>> {
152    let mut out = Vec::new();
153    for page in 1..=50 {
154        let sep = if url.contains('?') { '&' } else { '?' };
155        let mut req = client.get(format!("{url}{sep}per_page=100&page={page}"));
156        if let Some(a) = auth {
157            req = if gitlab {
158                req.header("PRIVATE-TOKEN", &a.token)
159            } else {
160                req.bearer_auth(&a.token)
161            };
162        }
163        let resp = req.send().await.with_context(|| format!("reach {host}"))?;
164        let status = resp.status().as_u16();
165        if status == 404 && page == 1 {
166            return Ok(None);
167        }
168        if status == 401 || status == 403 {
169            return Err(NeedsToken {
170                host: host.to_string(),
171                username: crate::worktree::token_user(host).into(),
172            }
173            .into());
174        }
175        if !resp.status().is_success() {
176            bail!("{host} answered {status}");
177        }
178        let items: Vec<serde_json::Value> = resp.json().await?;
179        let done = items.len() < 100;
180        out.extend(items);
181        if done {
182            break;
183        }
184    }
185    Ok(Some(out))
186}
187
188async fn github(
189    client: &reqwest::Client,
190    s: &Source,
191    auth: Option<&Auth>,
192) -> Result<Vec<RemoteRepo>> {
193    let api = github_api(&s.host);
194    let org = format!("{api}/orgs/{}/repos?type=all", s.owner);
195    let items = match pages(client, &org, auth, false, &s.host).await? {
196        Some(items) => items,
197        None => {
198            let user = format!("{api}/users/{}/repos?type=owner", s.owner);
199            pages(client, &user, auth, false, &s.host)
200                .await?
201                .with_context(|| format!("no organisation or user {} on {}", s.owner, s.host))?
202        }
203    };
204    Ok(items.iter().filter_map(github_repo).collect())
205}
206
207fn github_repo(v: &serde_json::Value) -> Option<RemoteRepo> {
208    #[derive(Deserialize)]
209    struct R {
210        name: String,
211        full_name: String,
212        clone_url: String,
213        description: Option<String>,
214        default_branch: Option<String>,
215        #[serde(default)]
216        archived: bool,
217        #[serde(default)]
218        fork: bool,
219        pushed_at: Option<String>,
220        size: Option<u64>,
221    }
222    let r: R = serde_json::from_value(v.clone()).ok()?;
223    Some(RemoteRepo {
224        name: r.name,
225        full_path: r.full_name,
226        clone_url: r.clone_url,
227        description: r.description.filter(|d| !d.is_empty()),
228        default_branch: r.default_branch,
229        archived: r.archived,
230        fork: r.fork,
231        last_activity: r.pushed_at,
232        size_kib: r.size,
233    })
234}
235
236async fn gitlab(
237    client: &reqwest::Client,
238    s: &Source,
239    auth: Option<&Auth>,
240) -> Result<Vec<RemoteRepo>> {
241    let id = s.owner.replace('/', "%2F");
242    let group = format!(
243        "https://{}/api/v4/groups/{id}/projects?include_subgroups=true&order_by=last_activity_at",
244        s.host
245    );
246    let items = match pages(client, &group, auth, true, &s.host).await? {
247        Some(items) => items,
248        None => {
249            let user = format!("https://{}/api/v4/users/{id}/projects", s.host);
250            pages(client, &user, auth, true, &s.host)
251                .await?
252                .with_context(|| format!("no group or user {} on {}", s.owner, s.host))?
253        }
254    };
255    Ok(items.iter().filter_map(gitlab_repo).collect())
256}
257
258fn gitlab_repo(v: &serde_json::Value) -> Option<RemoteRepo> {
259    #[derive(Deserialize)]
260    struct R {
261        path: String,
262        path_with_namespace: String,
263        http_url_to_repo: String,
264        description: Option<String>,
265        default_branch: Option<String>,
266        #[serde(default)]
267        archived: bool,
268        forked_from_project: Option<serde_json::Value>,
269        last_activity_at: Option<String>,
270    }
271    let r: R = serde_json::from_value(v.clone()).ok()?;
272    Some(RemoteRepo {
273        name: r.path,
274        full_path: r.path_with_namespace,
275        clone_url: r.http_url_to_repo,
276        description: r.description.filter(|d| !d.is_empty()),
277        default_branch: r.default_branch,
278        archived: r.archived,
279        fork: r.forked_from_project.is_some(),
280        last_activity: r.last_activity_at,
281        size_kib: None,
282    })
283}
284
285/// Where comindly keeps its own clone of `clone_url`: `<data>/repos/<host>/<path>`.
286pub fn managed_path(data_dir: &std::path::Path, clone_url: &str) -> Result<std::path::PathBuf> {
287    let rest = clone_url
288        .trim_start_matches("https://")
289        .trim_start_matches("http://")
290        .trim_start_matches("ssh://")
291        .trim_start_matches("git@");
292    let rest = rest.split_once('@').map_or(rest, |(_, r)| r);
293    let (host, path) = rest
294        .split_once(['/', ':'])
295        .context("cannot tell the host from the URL")?;
296    let path = path.trim_end_matches(".git").trim_matches('/');
297    anyhow::ensure!(
298        !host.is_empty() && !path.is_empty() && !path.contains(".."),
299        "cannot tell the repository from {clone_url}"
300    );
301    Ok(data_dir
302        .join("repos")
303        .join(host.split(':').next().unwrap_or(host))
304        .join(path))
305}
306
307/// A browser link to `path` at `line` in `commit` of the repository cloned from `remote`.
308pub fn file_link(remote: &str, commit: &str, path: &str, line: usize) -> Option<String> {
309    let (provider, base) = web_base(remote)?;
310    let path: String = path
311        .chars()
312        .map(|c| match c {
313            ' ' => "%20".to_string(),
314            '#' => "%23".to_string(),
315            '?' => "%3F".to_string(),
316            c => c.to_string(),
317        })
318        .collect();
319    Some(match provider {
320        Provider::GitLab => format!("{base}/-/blob/{commit}/{path}#L{line}"),
321        Provider::GitHub => format!("{base}/blob/{commit}/{path}#L{line}"),
322    })
323}
324
325/// A browser link to `commit` of the repository cloned from `remote`.
326pub fn commit_link(remote: &str, commit: &str) -> Option<String> {
327    let (provider, base) = web_base(remote)?;
328    Some(match provider {
329        Provider::GitLab => format!("{base}/-/commit/{commit}"),
330        Provider::GitHub => format!("{base}/commit/{commit}"),
331    })
332}
333
334/// The repository's web address, without any credentials the clone URL carries.
335fn web_base(remote: &str) -> Option<(Provider, String)> {
336    let s = parse(&crate::storage::without_userinfo(remote)).ok()?;
337    let path = match (&s.repo, s.provider) {
338        (Some(r), _) => format!("{}/{r}", s.owner),
339        (None, Provider::GitLab) => s.owner.clone(),
340        (None, Provider::GitHub) => return None,
341    };
342    Some((s.provider, format!("https://{}/{path}", s.host)))
343}
344
345#[cfg(test)]
346mod tests {
347    use super::*;
348
349    const DATA: &str = "/data/comindly";
350
351    #[test]
352    fn reads_org_and_group_urls() {
353        let s = parse("https://github.com/acme").unwrap();
354        assert_eq!(
355            (s.provider, s.host.as_str(), s.owner.as_str()),
356            (Provider::GitHub, "github.com", "acme")
357        );
358        assert_eq!(
359            parse("github.com/orgs/acme/repositories").unwrap().owner,
360            "acme"
361        );
362        let g = parse("https://gitlab.com/acme/platform/-/tree/main").unwrap();
363        assert_eq!(
364            (g.provider, g.owner.as_str()),
365            (Provider::GitLab, "acme/platform")
366        );
367        let own = parse("git.example.com/team/").unwrap();
368        assert_eq!(
369            (own.provider, own.host.as_str(), own.owner.as_str()),
370            (Provider::GitLab, "git.example.com", "team")
371        );
372        assert!(parse("github.com").is_err());
373        let r = parse("https://github.com/acme/orders").unwrap();
374        assert_eq!(
375            (r.owner.as_str(), r.repo.as_deref()),
376            ("acme", Some("orders"))
377        );
378        let ssh = parse("git@gitlab.com:acme/platform/payments.git").unwrap();
379        assert_eq!(
380            (ssh.host.as_str(), ssh.owner.as_str(), ssh.repo.as_deref()),
381            ("gitlab.com", "acme/platform", Some("payments"))
382        );
383        assert_eq!(parse("gitlab.com/acme/platform").unwrap().repo, None);
384    }
385
386    #[test]
387    fn reads_api_answers() {
388        let gh = serde_json::json!({"name": "orders", "full_name": "acme/orders", "clone_url": "https://github.com/acme/orders.git",
389            "description": "", "default_branch": "trunk", "archived": false, "fork": false, "pushed_at": "2026-10-01T00:00:00Z", "size": 2048});
390        let r = github_repo(&gh).unwrap();
391        assert_eq!(
392            (
393                r.full_path.as_str(),
394                r.default_branch.as_deref(),
395                r.description
396            ),
397            ("acme/orders", Some("trunk"), None)
398        );
399        let gl = serde_json::json!({"path": "payments", "path_with_namespace": "acme/platform/payments",
400            "http_url_to_repo": "https://gitlab.com/acme/platform/payments.git", "description": "Charges", "default_branch": "master",
401            "archived": true, "last_activity_at": "2026-09-01T00:00:00Z"});
402        let r = gitlab_repo(&gl).unwrap();
403        assert_eq!(
404            (r.name.as_str(), r.archived, r.fork),
405            ("payments", true, false)
406        );
407    }
408
409    #[test]
410    fn links_to_files_and_commits_without_credentials() {
411        assert_eq!(
412            file_link(
413                "https://oauth2:secret@gitlab.com/acme/team/orders.git",
414                "abc",
415                "src/a b.py",
416                7
417            )
418            .unwrap(),
419            "https://gitlab.com/acme/team/orders/-/blob/abc/src/a%20b.py#L7"
420        );
421        assert_eq!(
422            commit_link("git@github.com:acme/orders.git", "abc").unwrap(),
423            "https://github.com/acme/orders/commit/abc"
424        );
425        assert_eq!(
426            commit_link("https://git.example.com/acme/orders", "abc").unwrap(),
427            "https://git.example.com/acme/orders/-/commit/abc"
428        );
429        assert_eq!(commit_link("https://github.com/acme", "abc"), None);
430    }
431
432    #[test]
433    fn managed_paths_follow_host_and_group() {
434        let p = managed_path(
435            std::path::Path::new(DATA),
436            "https://gitlab.com/acme/platform/payments.git",
437        )
438        .unwrap();
439        assert!(
440            p.ends_with("repos/gitlab.com/acme/platform/payments"),
441            "{}",
442            p.display()
443        );
444        let s = managed_path(std::path::Path::new(DATA), "git@github.com:acme/orders.git").unwrap();
445        assert!(
446            s.ends_with("repos/github.com/acme/orders"),
447            "{}",
448            s.display()
449        );
450        assert!(managed_path(std::path::Path::new(DATA), "https://github.com/../etc").is_err());
451    }
452}
comindly_core/hosting.rs

comindly_core/
hosting.rs
