wsb_github/
checkout.rs

1//! What one checkout holds: the GitHub repositories in it, and the commit each
2//! is checked out at.
3//!
4//! Read from git and from the declarations git reads — the superproject's
5//! remote and every `.gitmodules` in the nesting tree — never from a list kept
6//! beside them. A submodule added this morning has issues too, and one removed
7//! stops being asked about.
8
9use std::collections::BTreeMap;
10use std::path::{Path, PathBuf};
11use std::process::Command;
12
13use anyhow::{bail, Context, Result};
14
15use crate::client::parse_github_url;
16
17/// One repository of a checkout.
18#[derive(Debug, Clone, PartialEq, Eq)]
19pub struct Repository {
20    /// `owner/repo` on GitHub.
21    pub name: String,
22    /// Where it is checked out in this checkout.
23    pub path: PathBuf,
24    /// The commit it is on, when it is checked out at all. A submodule that
25    /// was never initialised is declared and has no commit.
26    pub head: Option<String>,
27}
28
29/// A git command aimed only by the directory it runs in.
30///
31/// The variables git reads to decide WHICH repository it is acting on are
32/// cleared: a caller running inside a hook, or inside another git command,
33/// would otherwise aim every one of these at the caller's repository.
34fn git(dir: &Path) -> Command {
35    let mut command = Command::new("git");
36    command.arg("-C").arg(dir);
37    for variable in [
38        "GIT_DIR",
39        "GIT_WORK_TREE",
40        "GIT_INDEX_FILE",
41        "GIT_COMMON_DIR",
42        "GIT_OBJECT_DIRECTORY",
43        "GIT_ALTERNATE_OBJECT_DIRECTORIES",
44        "GIT_NAMESPACE",
45        "GIT_PREFIX",
46    ] {
47        command.env_remove(variable);
48    }
49    command
50}
51
52fn output(mut command: Command, what: &str) -> Result<std::process::Output> {
53    command
54        .output()
55        .with_context(|| format!("cannot run git to {what}"))
56}
57
58/// The URL of a repository's `origin`, or `None` when it has none.
59pub fn origin_url(dir: &Path) -> Result<Option<String>> {
60    let mut command = git(dir);
61    command.args(["remote", "get-url", "origin"]);
62    let out = output(command, &format!("read the origin of {}", dir.display()))?;
63    if out.status.success() {
64        Ok(Some(String::from_utf8_lossy(&out.stdout).trim().to_string()))
65    } else {
66        let stderr = String::from_utf8_lossy(&out.stderr);
67        // "No such remote" is an answer: this repository has no origin.
68        if stderr.contains("No such remote") {
69            Ok(None)
70        } else {
71            bail!("git could not read the origin of {}: {}", dir.display(), stderr.trim())
72        }
73    }
74}
75
76/// The commit a checkout is on, or `None` when it has none yet.
77///
78/// A repository with no commits — HEAD names a branch nothing is on — is a
79/// state, not a failure: there is simply no commit to ask CI about.
80pub fn head(dir: &Path) -> Result<Option<String>> {
81    let mut command = git(dir);
82    command.args(["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]);
83    let out = output(command, &format!("read the commit of {}", dir.display()))?;
84    match out.status.code() {
85        Some(0) => Ok(Some(String::from_utf8_lossy(&out.stdout).trim().to_string())),
86        Some(1) if out.stdout.is_empty() => Ok(None),
87        _ => bail!(
88            "git could not read the commit of {}: {}",
89            dir.display(),
90            String::from_utf8_lossy(&out.stderr).trim()
91        ),
92    }
93}
94
95/// Whether a folder is a checkout of its own: a repository or a worktree,
96/// whose `.git` is a folder or a file pointing at one.
97fn is_checked_out(dir: &Path) -> bool {
98    dir.join(".git").exists()
99}
100
101/// The submodules one repository declares: name, path and URL.
102fn declared_submodules(dir: &Path) -> Result<Vec<(String, String)>> {
103    if !dir.join(".gitmodules").is_file() {
104        return Ok(Vec::new());
105    }
106    let mut command = git(dir);
107    command.args([
108        "config",
109        "--file",
110        ".gitmodules",
111        "--get-regexp",
112        r"^submodule\..*\.(path|url)$",
113    ]);
114    let out = output(command, &format!("read {}/.gitmodules", dir.display()))?;
115    match out.status.code() {
116        Some(0) => {}
117        // `--get-regexp` answers 1 when nothing matched: a .gitmodules that
118        // declares no submodules.
119        Some(1) => return Ok(Vec::new()),
120        _ => bail!(
121            "git could not read {}/.gitmodules: {}",
122            dir.display(),
123            String::from_utf8_lossy(&out.stderr).trim()
124        ),
125    }
126    let mut paths: BTreeMap<String, String> = BTreeMap::new();
127    let mut urls: BTreeMap<String, String> = BTreeMap::new();
128    for line in String::from_utf8_lossy(&out.stdout).lines() {
129        let Some((key, value)) = line.split_once(' ') else {
130            continue;
131        };
132        let Some(rest) = key.strip_prefix("submodule.") else {
133            continue;
134        };
135        if let Some(name) = rest.strip_suffix(".path") {
136            paths.insert(name.to_string(), value.to_string());
137        } else if let Some(name) = rest.strip_suffix(".url") {
138            urls.insert(name.to_string(), value.to_string());
139        }
140    }
141    let mut declared = Vec::new();
142    for (name, path) in paths {
143        let url = urls.remove(&name).ok_or_else(|| {
144            anyhow::anyhow!(
145                "{}/.gitmodules declares submodule {name:?} with a path and no url",
146                dir.display()
147            )
148        })?;
149        declared.push((path, url));
150    }
151    Ok(declared)
152}
153
154/// A submodule URL as git resolves it: a relative one is relative to the URL
155/// of the repository that declares it.
156fn resolve_url(declared: &str, parent: Option<&str>) -> Option<String> {
157    if !(declared.starts_with("./") || declared.starts_with("../")) {
158        return Some(declared.to_string());
159    }
160    let parent = parent?.trim_end_matches('/').trim_end_matches(".git");
161    // Split the parent into the part before its path and its path segments,
162    // for the two forms git writes: `scheme://host/path` and `host:path`.
163    let (prefix, path) = if let Some(index) = parent.find("://") {
164        let after = &parent[index + 3..];
165        let slash = after.find('/')?;
166        (&parent[..index + 3 + slash + 1], &after[slash + 1..])
167    } else {
168        let colon = parent.find(':')?;
169        (&parent[..colon + 1], &parent[colon + 1..])
170    };
171    let mut segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
172    for part in declared.split('/') {
173        match part {
174            "." | "" => {}
175            ".." => {
176                segments.pop()?;
177            }
178            other => segments.push(other),
179        }
180    }
181    Some(format!("{prefix}{}", segments.join("/")))
182}
183
184/// Every GitHub repository in a checkout: the superproject and every
185/// submodule in its nesting tree, parent before child.
186///
187/// A submodule's repository is the one its declaration names, whether or not
188/// it is checked out: a repository declared and never initialised still has
189/// issues. Its own submodules are known only once it is checked out, because
190/// that is where they are declared.
191pub fn repositories(root: &Path) -> Result<Vec<Repository>> {
192    let mut found = Vec::new();
193    let origin = origin_url(root)?;
194    if let Some((owner, repo)) = origin.as_deref().and_then(parse_github_url) {
195        found.push(Repository {
196            name: format!("{owner}/{repo}"),
197            path: root.to_path_buf(),
198            head: head(root)?,
199        });
200    }
201    walk(root, origin.as_deref(), &mut found)?;
202    Ok(found)
203}
204
205fn walk(dir: &Path, parent_url: Option<&str>, found: &mut Vec<Repository>) -> Result<()> {
206    for (path, declared_url) in declared_submodules(dir)? {
207        let child = dir.join(&path);
208        let url = resolve_url(&declared_url, parent_url);
209        let checked_out = is_checked_out(&child);
210        if let Some((owner, repo)) = url.as_deref().and_then(parse_github_url) {
211            found.push(Repository {
212                name: format!("{owner}/{repo}"),
213                path: child.clone(),
214                head: if checked_out { head(&child)? } else { None },
215            });
216        }
217        if checked_out {
218            walk(&child, url.as_deref(), found)?;
219        }
220    }
221    Ok(())
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227
228    fn run(dir: &Path, args: &[&str]) {
229        let out = git(dir).args(args).output().expect("git runs");
230        assert!(
231            out.status.success(),
232            "git {args:?} failed: {}",
233            String::from_utf8_lossy(&out.stderr)
234        );
235    }
236
237    /// A repository with one commit and, when given, an origin.
238    fn repository(dir: &Path, origin: Option<&str>) -> String {
239        std::fs::create_dir_all(dir).expect("a folder");
240        run(dir, &["init", "-q", "-b", "main"]);
241        run(dir, &["config", "user.email", "wsb-github-tests@example.com"]);
242        run(dir, &["config", "user.name", "wsb-github tests"]);
243        std::fs::write(dir.join("README.md"), "a repository\n").expect("a file");
244        run(dir, &["add", "README.md"]);
245        run(dir, &["commit", "-q", "-m", "init"]);
246        if let Some(origin) = origin {
247            run(dir, &["remote", "add", "origin", origin]);
248        }
249        head(dir).expect("readable").expect("a commit")
250    }
251
252    #[test]
253    fn a_checkout_names_every_github_repository_in_its_tree() {
254        let temp = tempfile::tempdir().expect("a scratch directory");
255        let root = temp.path().join("workspace");
256        let root_head = repository(&root, Some("git@github.com:example/workspace.git"));
257        std::fs::write(
258            root.join(".gitmodules"),
259            "[submodule \"core\"]\n\tpath = libs/core\n\turl = ../core.git\n\
260             [submodule \"data\"]\n\tpath = data\n\turl = https://huggingface.co/datasets/example/data\n\
261             [submodule \"later\"]\n\tpath = libs/later\n\turl = https://github.com/example/later\n",
262        )
263        .expect("declared");
264        // `core` is checked out and declares a submodule of its own; `later`
265        // is declared and was never initialised.
266        let core = root.join("libs/core");
267        let core_head = repository(&core, Some("git@github.com:example/core.git"));
268        std::fs::write(
269            core.join(".gitmodules"),
270            "[submodule \"nested\"]\n\tpath = nested\n\turl = git@github.com:example/nested.git\n",
271        )
272        .expect("declared");
273
274        let found = repositories(&root).expect("read");
275        let named: Vec<(&str, Option<&str>)> = found
276            .iter()
277            .map(|repository| (repository.name.as_str(), repository.head.as_deref()))
278            .collect();
279        assert_eq!(
280            named,
281            vec![
282                ("example/workspace", Some(root_head.as_str())),
283                // Relative to the workspace's own URL, as git resolves it.
284                ("example/core", Some(core_head.as_str())),
285                // Declared inside a submodule, so known because it is checked out.
286                ("example/nested", None),
287                // Declared and never initialised: a repository, with no commit.
288                ("example/later", None),
289            ],
290            "the Hugging Face submodule is not on GitHub and must not be listed"
291        );
292    }
293
294    #[test]
295    fn a_superproject_off_github_still_has_its_github_submodules() {
296        let temp = tempfile::tempdir().expect("a scratch directory");
297        let root = temp.path().join("workspace");
298        repository(&root, Some("https://huggingface.co/datasets/example/workspace"));
299        std::fs::write(
300            root.join(".gitmodules"),
301            "[submodule \"core\"]\n\tpath = core\n\turl = git@github.com:example/core.git\n",
302        )
303        .expect("declared");
304        let names: Vec<String> = repositories(&root)
305            .expect("read")
306            .into_iter()
307            .map(|repository| repository.name)
308            .collect();
309        assert_eq!(names, vec!["example/core".to_string()]);
310    }
311
312    #[test]
313    fn a_repository_with_no_origin_and_no_commit_is_read_not_refused() {
314        let temp = tempfile::tempdir().expect("a scratch directory");
315        let root = temp.path().join("fresh");
316        std::fs::create_dir_all(&root).expect("a folder");
317        run(&root, &["init", "-q", "-b", "main"]);
318        assert_eq!(head(&root).expect("readable"), None, "an unborn HEAD has no commit");
319        assert_eq!(origin_url(&root).expect("readable"), None);
320        assert!(repositories(&root).expect("read").is_empty());
321    }
322
323    /// TEST12911: a submodule is found by the repository its declaration names,
324    /// whatever its clone's `origin` still says.
325    ///
326    /// A repository renamed on GitHub after it was cloned keeps the old URL in
327    /// its clone, which GitHub redirects, while the declaration has the new
328    /// one. Looked up by `origin`, its checks were "not checked out in this
329    /// workspace" in a workspace that had it checked out (18 Oct 2026).
330    #[test]
331    fn test12911_a_renamed_submodule_is_found_by_its_declared_name() {
332        let temp = tempfile::tempdir().expect("a scratch directory");
333        let root = temp.path().join("workspace");
334        repository(&root, Some("https://github.com/example/workspace.git"));
335        std::fs::write(
336            root.join(".gitmodules"),
337            "[submodule \"core\"]\n\tpath = libs/core\n\turl = https://github.com/example/core.git\n",
338        )
339        .expect("declared");
340        let head = repository(&root.join("libs/core"), Some("https://github.com/example/core-old.git"));
341
342        let found = repositories(&root).expect("read");
343        let core = found
344            .iter()
345            .find(|repository| repository.name == "example/core")
346            .expect("the declared name finds the checkout");
347        assert_eq!(
348            core.head.as_deref(),
349            Some(head.as_str()),
350            "the checks of a submodule are asked at the commit its checkout is on"
351        );
352        assert!(
353            !found.iter().any(|repository| repository.name == "example/core-old"),
354            "the name the clone was made under is not one the workspace declares: {found:?}"
355        );
356    }
357
358    #[test]
359    fn a_relative_submodule_url_resolves_against_the_declaring_repository() {
360        assert_eq!(
361            resolve_url("../core.git", Some("https://github.com/example/workspace.git")).as_deref(),
362            Some("https://github.com/example/core.git")
363        );
364        assert_eq!(
365            resolve_url("../core", Some("git@github.com:example/workspace")).as_deref(),
366            Some("git@github.com:example/core")
367        );
368        assert_eq!(
369            resolve_url("./nested", Some("https://github.com/example/workspace")).as_deref(),
370            Some("https://github.com/example/workspace/nested")
371        );
372        // An absolute URL is taken as it stands, parent or not.
373        assert_eq!(
374            resolve_url("https://github.com/other/thing", None).as_deref(),
375            Some("https://github.com/other/thing")
376        );
377        // A relative URL with nothing to be relative to cannot be resolved,
378        // and is not guessed at.
379        assert_eq!(resolve_url("../core", None), None);
380        // Nor can one that climbs above the host.
381        assert_eq!(
382            resolve_url("../../../x", Some("https://github.com/example/workspace")),
383            None
384        );
385    }
386}
wsb_github/checkout.rs

wsb_github/
checkout.rs
