comindly_core/
hosting.rs

1//! Lists the repositories of a GitHub organisation or user, or a GitLab group (with subgroups), so
2//! they can be picked and cloned. Works with github.com, GitHub Enterprise, gitlab.com and self-hosted GitLab.
3
4use anyhow::{bail, Context, Result};
5use serde::{Deserialize, Serialize};
6
7use crate::worktree::{Auth, NeedsToken};
8
9/// Where the repositories live: a provider, its host and the owner or group path.
10#[derive(Debug, Clone, PartialEq)]
11pub struct Source {
12    pub provider: Provider,
13    pub host: String,
14    /// Organisation or user on GitHub; group or project path (`team/platform`) on GitLab.
15    pub owner: String,
16    /// The repository, when the URL names one (`github.com/acme/orders`, `….git`, `git@…`).
17    pub repo: Option<String>,
18}
19
20#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22pub enum Provider {
23    GitHub,
24    GitLab,
25}
26
27/// A repository on the host.
28#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
29pub struct RemoteRepo {
30    pub name: String,
31    /// `owner/name` or `group/sub/name`.
32    pub full_path: String,
33    pub clone_url: String,
34    pub description: Option<String>,
35    pub default_branch: Option<String>,
36    pub archived: bool,
37    pub fork: bool,
38    pub last_activity: Option<String>,
39    /// Size in KiB, when the host reports it.
40    pub size_kib: Option<u64>,
41}
42
43/// Read `https://github.com/acme`, `gitlab.com/acme/platform` or `https://git.example.com/group`.
44pub fn parse(url: &str) -> Result<Source> {
45    let url = url.trim().trim_end_matches('/');
46    let clone_url = url.ends_with(".git") || url.starts_with("git@") || url.starts_with("ssh://");
47    let rest = url
48        .trim_start_matches("https://")
49        .trim_start_matches("http://")
50        .trim_start_matches("ssh://")
51        .trim_start_matches("git@");
52    let rest = rest.split(['?', '#']).next().unwrap_or_default();
53    let (host, path) = rest
54        .split_once(['/', ':'])
55        .context("give the organisation, group or repository, e.g. github.com/acme")?;
56    let path = path.trim_end_matches(".git").trim_matches('/');
57    let path = path.split("/-/").next().unwrap_or(path);
58    anyhow::ensure!(
59        !path.is_empty(),
60        "give the organisation, group or repository after the host"
61    );
62    let host = host.to_lowercase();
63    let provider = if host == "github.com" || host.contains("github") {
64        Provider::GitHub
65    } else {
66        Provider::GitLab
67    };
68    let parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
69    let (owner, repo) = match provider {
70        // github.com/orgs/acme/repositories is the organisation; github.com/acme/orders a repository.
71        Provider::GitHub if parts.first() == Some(&"orgs") => {
72            (parts.get(1).unwrap_or(&"").to_string(), None)
73        }
74        Provider::GitHub => (parts[0].to_string(), parts.get(1).map(|r| r.to_string())),
75        Provider::GitLab if clone_url && parts.len() >= 2 => {
76            let (group, name) = path.rsplit_once('/').unwrap_or_default();
77            (group.to_string(), Some(name.to_string()))
78        }
79        Provider::GitLab => (path.to_string(), None),
80    };
81    anyhow::ensure!(
82        !owner.is_empty(),
83        "give the organisation, group or repository after the host"
84    );
85    Ok(Source {
86        provider,
87        host,
88        owner,
89        repo,
90    })
91}
92
93/// Every repository of `source`, newest activity first.
94pub async fn list(source: &Source, auth: Option<&Auth>) -> Result<Vec<RemoteRepo>> {
95    let client = reqwest::Client::builder()
96        .user_agent(concat!("comindly/", env!("CARGO_PKG_VERSION")))
97        .build()?;
98    let mut repos = match source.provider {
99        Provider::GitHub => match &source.repo {
100            Some(name) => {
101                let api = github_api(&source.host);
102                let url = format!("{api}/repos/{}/{name}", source.owner);
103                let one = one(&client, &url, auth, false, &source.host)
104                    .await?
105                    .with_context(|| {
106                        format!("no repository {}/{name} on {}", source.owner, source.host)
107                    })?;
108                github_repo(&one).into_iter().collect()
109            }
110            None => github(&client, source, auth).await?,
111        },
112        Provider::GitLab => {
113            // `a/b` is a project or a subgroup; ask for the project first.
114            let path = match &source.repo {
115                Some(name) => format!("{}/{name}", source.owner),
116                None => source.owner.clone(),
117            };
118            let url = format!(
119                "https://{}/api/v4/projects/{}",
120                source.host,
121                path.replace('/', "%2F")
122            );
123            let project = if path.contains('/') {
124                one(&client, &url, auth, true, &source.host).await?
125            } else {
126                None
127            };
128            match project {
129                Some(p) => gitlab_repo(&p).into_iter().collect(),
130                None if source.repo.is_some() => bail!("no repository {path} on {}", source.host),
131                None => gitlab(&client, source, auth).await?,
132            }
133        }
134    };
135    repos.sort_by(|a, b| b.last_activity.cmp(&a.last_activity));
136    Ok(repos)
137}
138
139/// One object, or `None` when it is not found.
140async fn one(
141    client: &reqwest::Client,
142    url: &str,
143    auth: Option<&Auth>,
144    gitlab: bool,
145    host: &str,
146) -> Result<Option<serde_json::Value>> {
147    let mut req = client.get(url);
148    if let Some(a) = auth {
149        req = if gitlab {
150            req.header("PRIVATE-TOKEN", &a.token)
151        } else {
152            req.bearer_auth(&a.token)
153        };
154    }
155    let resp = req.send().await.with_context(|| format!("reach {host}"))?;
156    match resp.status().as_u16() {
157        404 => Ok(None),
158        401 | 403 => Err(NeedsToken {
159            host: host.to_string(),
160            username: crate::worktree::token_user(host).into(),
161        }
162        .into()),
163        s if !resp.status().is_success() => bail!("{host} answered {s}"),
164        _ => Ok(Some(resp.json().await?)),
165    }
166}
167
168fn github_api(host: &str) -> String {
169    if host == "github.com" {
170        "https://api.github.com".to_string()
171    } else {
172        format!("https://{host}/api/v3")
173    }
174}
175
176/// Fetch every page of a listing; `None` when the first page is not found.
177async fn pages(
178    client: &reqwest::Client,
179    url: &str,
180    auth: Option<&Auth>,
181    gitlab: bool,
182    host: &str,
183) -> Result<Option<Vec<serde_json::Value>>> {
184    let mut out = Vec::new();
185    for page in 1..=50 {
186        let sep = if url.contains('?') { '&' } else { '?' };
187        let mut req = client.get(format!("{url}{sep}per_page=100&page={page}"));
188        if let Some(a) = auth {
189            req = if gitlab {
190                req.header("PRIVATE-TOKEN", &a.token)
191            } else {
192                req.bearer_auth(&a.token)
193            };
194        }
195        let resp = req.send().await.with_context(|| format!("reach {host}"))?;
196        let status = resp.status().as_u16();
197        if status == 404 && page == 1 {
198            return Ok(None);
199        }
200        if status == 401 || status == 403 {
201            return Err(NeedsToken {
202                host: host.to_string(),
203                username: crate::worktree::token_user(host).into(),
204            }
205            .into());
206        }
207        if !resp.status().is_success() {
208            bail!("{host} answered {status}");
209        }
210        let items: Vec<serde_json::Value> = resp.json().await?;
211        let done = items.len() < 100;
212        out.extend(items);
213        if done {
214            break;
215        }
216    }
217    Ok(Some(out))
218}
219
220async fn github(
221    client: &reqwest::Client,
222    s: &Source,
223    auth: Option<&Auth>,
224) -> Result<Vec<RemoteRepo>> {
225    let api = github_api(&s.host);
226    let org = format!("{api}/orgs/{}/repos?type=all", s.owner);
227    let items = match pages(client, &org, auth, false, &s.host).await? {
228        Some(items) => items,
229        None => {
230            let user = format!("{api}/users/{}/repos?type=owner", s.owner);
231            pages(client, &user, auth, false, &s.host)
232                .await?
233                .with_context(|| format!("no organisation or user {} on {}", s.owner, s.host))?
234        }
235    };
236    Ok(items.iter().filter_map(github_repo).collect())
237}
238
239fn github_repo(v: &serde_json::Value) -> Option<RemoteRepo> {
240    #[derive(Deserialize)]
241    struct R {
242        name: String,
243        full_name: String,
244        clone_url: String,
245        description: Option<String>,
246        default_branch: Option<String>,
247        #[serde(default)]
248        archived: bool,
249        #[serde(default)]
250        fork: bool,
251        pushed_at: Option<String>,
252        size: Option<u64>,
253    }
254    let r: R = serde_json::from_value(v.clone()).ok()?;
255    Some(RemoteRepo {
256        name: r.name,
257        full_path: r.full_name,
258        clone_url: r.clone_url,
259        description: r.description.filter(|d| !d.is_empty()),
260        default_branch: r.default_branch,
261        archived: r.archived,
262        fork: r.fork,
263        last_activity: r.pushed_at,
264        size_kib: r.size,
265    })
266}
267
268async fn gitlab(
269    client: &reqwest::Client,
270    s: &Source,
271    auth: Option<&Auth>,
272) -> Result<Vec<RemoteRepo>> {
273    let id = s.owner.replace('/', "%2F");
274    let group = format!(
275        "https://{}/api/v4/groups/{id}/projects?include_subgroups=true&order_by=last_activity_at",
276        s.host
277    );
278    let items = match pages(client, &group, auth, true, &s.host).await? {
279        Some(items) => items,
280        None => {
281            let user = format!("https://{}/api/v4/users/{id}/projects", s.host);
282            pages(client, &user, auth, true, &s.host)
283                .await?
284                .with_context(|| format!("no group or user {} on {}", s.owner, s.host))?
285        }
286    };
287    Ok(items.iter().filter_map(gitlab_repo).collect())
288}
289
290fn gitlab_repo(v: &serde_json::Value) -> Option<RemoteRepo> {
291    #[derive(Deserialize)]
292    struct R {
293        path: String,
294        path_with_namespace: String,
295        http_url_to_repo: String,
296        description: Option<String>,
297        default_branch: Option<String>,
298        #[serde(default)]
299        archived: bool,
300        forked_from_project: Option<serde_json::Value>,
301        last_activity_at: Option<String>,
302    }
303    let r: R = serde_json::from_value(v.clone()).ok()?;
304    Some(RemoteRepo {
305        name: r.path,
306        full_path: r.path_with_namespace,
307        clone_url: r.http_url_to_repo,
308        description: r.description.filter(|d| !d.is_empty()),
309        default_branch: r.default_branch,
310        archived: r.archived,
311        fork: r.forked_from_project.is_some(),
312        last_activity: r.last_activity_at,
313        size_kib: None,
314    })
315}
316
317/// Where comindly keeps its own clone of `clone_url`: `<data>/repos/<host>/<path>`.
318pub fn managed_path(data_dir: &std::path::Path, clone_url: &str) -> Result<std::path::PathBuf> {
319    let rest = clone_url
320        .trim_start_matches("https://")
321        .trim_start_matches("http://")
322        .trim_start_matches("ssh://")
323        .trim_start_matches("git@");
324    let rest = rest.split_once('@').map_or(rest, |(_, r)| r);
325    let (host, path) = rest
326        .split_once(['/', ':'])
327        .context("cannot tell the host from the URL")?;
328    let path = path.trim_end_matches(".git").trim_matches('/');
329    anyhow::ensure!(
330        !host.is_empty() && !path.is_empty() && !path.contains(".."),
331        "cannot tell the repository from {clone_url}"
332    );
333    Ok(data_dir
334        .join("repos")
335        .join(host.split(':').next().unwrap_or(host))
336        .join(path))
337}
338
339/// A browser link to `path` at `line` in `commit` of the repository cloned from `remote`.
340pub fn file_link(remote: &str, commit: &str, path: &str, line: usize) -> Option<String> {
341    let (provider, base) = web_base(remote)?;
342    let path: String = path
343        .chars()
344        .map(|c| match c {
345            ' ' => "%20".to_string(),
346            '#' => "%23".to_string(),
347            '?' => "%3F".to_string(),
348            c => c.to_string(),
349        })
350        .collect();
351    Some(match provider {
352        Provider::GitLab => format!("{base}/-/blob/{commit}/{path}#L{line}"),
353        Provider::GitHub => format!("{base}/blob/{commit}/{path}#L{line}"),
354    })
355}
356
357/// A browser link to `commit` of the repository cloned from `remote`.
358pub fn commit_link(remote: &str, commit: &str) -> Option<String> {
359    let (provider, base) = web_base(remote)?;
360    Some(match provider {
361        Provider::GitLab => format!("{base}/-/commit/{commit}"),
362        Provider::GitHub => format!("{base}/commit/{commit}"),
363    })
364}
365
366/// The repository's web address, without any credentials the clone URL carries.
367fn web_base(remote: &str) -> Option<(Provider, String)> {
368    let s = parse(&crate::storage::without_userinfo(remote)).ok()?;
369    let path = match (&s.repo, s.provider) {
370        (Some(r), _) => format!("{}/{r}", s.owner),
371        (None, Provider::GitLab) => s.owner.clone(),
372        (None, Provider::GitHub) => return None,
373    };
374    Some((s.provider, format!("https://{}/{path}", s.host)))
375}
376
377#[cfg(test)]
378mod tests {
379    use super::*;
380
381    const DATA: &str = "/data/comindly";
382
383    #[test]
384    fn reads_org_and_group_urls() {
385        let s = parse("https://github.com/acme").unwrap();
386        assert_eq!(
387            (s.provider, s.host.as_str(), s.owner.as_str()),
388            (Provider::GitHub, "github.com", "acme")
389        );
390        assert_eq!(
391            parse("github.com/orgs/acme/repositories").unwrap().owner,
392            "acme"
393        );
394        let g = parse("https://gitlab.com/acme/platform/-/tree/main").unwrap();
395        assert_eq!(
396            (g.provider, g.owner.as_str()),
397            (Provider::GitLab, "acme/platform")
398        );
399        let own = parse("git.example.com/team/").unwrap();
400        assert_eq!(
401            (own.provider, own.host.as_str(), own.owner.as_str()),
402            (Provider::GitLab, "git.example.com", "team")
403        );
404        assert!(parse("github.com").is_err());
405        let r = parse("https://github.com/acme/orders").unwrap();
406        assert_eq!(
407            (r.owner.as_str(), r.repo.as_deref()),
408            ("acme", Some("orders"))
409        );
410        let ssh = parse("git@gitlab.com:acme/platform/payments.git").unwrap();
411        assert_eq!(
412            (ssh.host.as_str(), ssh.owner.as_str(), ssh.repo.as_deref()),
413            ("gitlab.com", "acme/platform", Some("payments"))
414        );
415        assert_eq!(parse("gitlab.com/acme/platform").unwrap().repo, None);
416    }
417
418    #[test]
419    fn reads_api_answers() {
420        let gh = serde_json::json!({"name": "orders", "full_name": "acme/orders", "clone_url": "https://github.com/acme/orders.git",
421            "description": "", "default_branch": "trunk", "archived": false, "fork": false, "pushed_at": "2026-10-01T00:00:00Z", "size": 2048});
422        let r = github_repo(&gh).unwrap();
423        assert_eq!(
424            (
425                r.full_path.as_str(),
426                r.default_branch.as_deref(),
427                r.description
428            ),
429            ("acme/orders", Some("trunk"), None)
430        );
431        let gl = serde_json::json!({"path": "payments", "path_with_namespace": "acme/platform/payments",
432            "http_url_to_repo": "https://gitlab.com/acme/platform/payments.git", "description": "Charges", "default_branch": "master",
433            "archived": true, "last_activity_at": "2026-09-01T00:00:00Z"});
434        let r = gitlab_repo(&gl).unwrap();
435        assert_eq!(
436            (r.name.as_str(), r.archived, r.fork),
437            ("payments", true, false)
438        );
439    }
440
441    #[test]
442    fn links_to_files_and_commits_without_credentials() {
443        assert_eq!(
444            file_link(
445                "https://oauth2:secret@gitlab.com/acme/team/orders.git",
446                "abc",
447                "src/a b.py",
448                7
449            )
450            .unwrap(),
451            "https://gitlab.com/acme/team/orders/-/blob/abc/src/a%20b.py#L7"
452        );
453        assert_eq!(
454            commit_link("git@github.com:acme/orders.git", "abc").unwrap(),
455            "https://github.com/acme/orders/commit/abc"
456        );
457        assert_eq!(
458            commit_link("https://git.example.com/acme/orders", "abc").unwrap(),
459            "https://git.example.com/acme/orders/-/commit/abc"
460        );
461        assert_eq!(commit_link("https://github.com/acme", "abc"), None);
462    }
463
464    #[test]
465    fn managed_paths_follow_host_and_group() {
466        let p = managed_path(
467            std::path::Path::new(DATA),
468            "https://gitlab.com/acme/platform/payments.git",
469        )
470        .unwrap();
471        assert!(
472            p.ends_with("repos/gitlab.com/acme/platform/payments"),
473            "{}",
474            p.display()
475        );
476        let s = managed_path(std::path::Path::new(DATA), "git@github.com:acme/orders.git").unwrap();
477        assert!(
478            s.ends_with("repos/github.com/acme/orders"),
479            "{}",
480            s.display()
481        );
482        assert!(managed_path(std::path::Path::new(DATA), "https://github.com/../etc").is_err());
483    }
484}
comindly_core/hosting.rs

comindly_core/
hosting.rs
