use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use crate::config::{Config, VcsKind};
use crate::paths::Paths;
use crate::source::RepoRef;
use crate::state::State;
use crate::workdir::{self, WorkdirCache};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CanonicalRepo {
pub path: PathBuf,
pub vcs: VcsKind,
pub name: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OnMissing {
Ask,
Clone,
}
#[derive(Debug, thiserror::Error)]
#[error("no local checkout of `{url}` found (would clone into {})", dest.display())]
pub struct NeedsClone {
pub url: String,
pub dest: PathBuf,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct RegistryEntry {
path: PathBuf,
origin_url: String,
created: chrono::DateTime<chrono::Utc>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct Registry {
repos: BTreeMap<String, RegistryEntry>,
}
impl Registry {
fn load(path: &Path) -> Result<Self> {
if !path.exists() {
return Ok(Self::default());
}
let text =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
if text.trim().is_empty() {
return Ok(Self::default());
}
serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))
}
fn save(&self, path: &Path) -> Result<()> {
let dir = path
.parent()
.context("registry path has no parent directory")?;
std::fs::create_dir_all(dir)?;
let tmp = dir.join(format!(
".{}.tmp",
path.file_name().unwrap().to_string_lossy()
));
std::fs::write(&tmp, serde_json::to_string_pretty(self)?)
.with_context(|| format!("writing {}", tmp.display()))?;
std::fs::rename(&tmp, path)
.with_context(|| format!("renaming {} to {}", tmp.display(), path.display()))?;
Ok(())
}
}
const RESCAN_THROTTLE: chrono::Duration = chrono::Duration::seconds(60);
pub struct RepoStore {
registry_path: PathBuf,
registry: Registry,
repos_dir: PathBuf,
workdir: Option<PathBuf>,
workdir_cache_path: PathBuf,
workdir_cache: Option<WorkdirCache>,
}
impl RepoStore {
pub fn load(paths: &Paths, config: &Config) -> Result<Self> {
let workdir_cache_path = paths.workdir_cache_file();
Ok(Self {
registry_path: paths.repos_file(),
registry: Registry::load(&paths.repos_file())?,
repos_dir: paths.repos_dir(),
workdir: config.workdir.clone(),
workdir_cache: WorkdirCache::load(&workdir_cache_path)?,
workdir_cache_path,
})
}
pub fn resolve(&mut self, repo_ref: &RepoRef, on_missing: OnMissing) -> Result<CanonicalRepo> {
let normalized: Vec<String> = repo_ref.urls.iter().map(|u| normalize_url(u)).collect();
if let Some(canon) = self.lookup_workdir(&normalized) {
return Ok(canon);
}
for n in &normalized {
if let Some(entry) = self.registry.repos.get(n)
&& entry.path.exists()
{
return Ok(CanonicalRepo {
path: entry.path.clone(),
vcs: VcsKind::Git,
name: n.clone(),
});
}
}
self.rescan_if_needed()?;
if let Some(canon) = self.lookup_workdir(&normalized) {
return Ok(canon);
}
let url = repo_ref
.urls
.first()
.context("review's repo has no candidate URLs to clone")?;
let name = normalize_url(url);
let dest = self.repos_dir.join(&name);
if on_missing == OnMissing::Ask {
return Err(NeedsClone {
url: url.clone(),
dest,
}
.into());
}
if dest.exists() {
tracing::warn!(
"{} already exists but wasn't registered; reusing it as-is",
dest.display()
);
} else {
clone_repo(url, &dest)?;
}
self.registry.repos.insert(
name.clone(),
RegistryEntry {
path: dest.clone(),
origin_url: url.clone(),
created: chrono::Utc::now(),
},
);
Ok(CanonicalRepo {
path: dest,
vcs: VcsKind::Git,
name,
})
}
fn lookup_workdir(&self, normalized: &[String]) -> Option<CanonicalRepo> {
let cache = self.workdir_cache.as_ref()?;
if Some(&cache.workdir) != self.workdir.as_ref() {
return None;
}
let repo = cache.lookup(normalized)?;
Some(CanonicalRepo {
path: repo.path.clone(),
vcs: repo.vcs,
name: repo.name.clone(),
})
}
pub fn rescan_if_needed(&mut self) -> Result<()> {
if self.workdir.is_some() && self.needs_rescan() {
self.rescan()?;
}
Ok(())
}
fn needs_rescan(&self) -> bool {
match &self.workdir_cache {
None => true,
Some(cache) => {
Some(&cache.workdir) != self.workdir.as_ref()
|| chrono::Utc::now().signed_duration_since(cache.scanned_at) >= RESCAN_THROTTLE
}
}
}
fn rescan(&mut self) -> Result<()> {
let Some(workdir) = self.workdir.clone() else {
return Ok(());
};
let repos = workdir::scan(&workdir, &self.repos_dir);
let cache = WorkdirCache {
workdir,
scanned_at: chrono::Utc::now(),
repos,
};
cache.save(&self.workdir_cache_path)?;
self.workdir_cache = Some(cache);
Ok(())
}
pub fn save(&self) -> Result<()> {
self.registry.save(&self.registry_path)
}
pub fn list(&self, state: &State) -> Vec<RepoListEntry> {
let mut out = Vec::new();
if let Some(cache) = &self.workdir_cache {
for r in &cache.repos {
let workspace_count = state
.workspaces()
.filter(|(_, w)| w.repo_path == r.path)
.count();
out.push(RepoListEntry {
url: r.name.clone(),
path: r.path.clone(),
kind: RepoKind::Discovered,
workspace_count,
});
}
}
for (url, entry) in &self.registry.repos {
let workspace_count = state
.workspaces()
.filter(|(_, w)| w.repo_path == entry.path)
.count();
out.push(RepoListEntry {
url: url.clone(),
path: entry.path.clone(),
kind: RepoKind::ToolManaged,
workspace_count,
});
}
out
}
pub fn remove(&mut self, url: &str, state: &State) -> Result<()> {
let normalized = normalize_url(url);
if self
.workdir_cache
.as_ref()
.is_some_and(|cache| cache.repos.iter().any(|r| r.remotes.contains(&normalized)))
{
bail!(
"`{url}` is a discovered repo in your workdir; rq never deletes those - remove the checkout yourself if you want it forgotten"
);
}
let Some(entry) = self.registry.repos.get(&normalized).cloned() else {
bail!("no tool-managed clone registered for `{url}` (see `rq repo list`)");
};
if state.workspaces().any(|(_, w)| w.repo_path == entry.path) {
bail!(
"{} still has workspaces using it; wait for `rq sync` to clean up resolved \
ones, or remove them by hand first",
entry.path.display()
);
}
std::fs::remove_dir_all(&entry.path)
.with_context(|| format!("removing {}", entry.path.display()))?;
self.registry.repos.remove(&normalized);
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RepoKind {
Discovered,
ToolManaged,
}
#[derive(Debug, Clone)]
pub struct RepoListEntry {
pub url: String,
pub path: PathBuf,
pub kind: RepoKind,
pub workspace_count: usize,
}
fn clone_repo(url: &str, dest: &Path) -> Result<()> {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("creating {}", parent.display()))?;
}
let output = std::process::Command::new("git")
.args(["clone", "--filter=blob:none", url, &dest.to_string_lossy()])
.output()
.with_context(|| format!("cloning {url}"))?;
if !output.status.success() {
bail!(
"cloning {url} into {} failed: {}",
dest.display(),
String::from_utf8_lossy(&output.stderr).trim()
);
}
Ok(())
}
pub fn normalize_url(url: &str) -> String {
let url = url.trim().trim_end_matches(".git");
if let Some((host_part, path_part)) = url.split_once(':')
&& !host_part.contains('/')
&& host_part.contains('@')
{
let host = host_part.rsplit('@').next().unwrap_or(host_part);
return format!("{host}/{}", path_part.trim_matches('/'));
}
if let Some(idx) = url.find("://") {
let rest = &url[idx + 3..];
let rest = rest.split_once('@').map(|(_, h)| h).unwrap_or(rest);
return rest.trim_matches('/').to_string();
}
url.trim_matches('/').to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::paths::Paths;
use crate::workdir::DiscoveredRepo;
use std::process::Command;
use tempfile::TempDir;
#[test]
fn normalizes_equivalent_github_urls() {
let expected = "github.com/mozilla-firefox/firefox";
assert_eq!(
normalize_url("https://github.com/mozilla-firefox/firefox"),
expected
);
assert_eq!(
normalize_url("https://github.com/mozilla-firefox/firefox.git"),
expected
);
assert_eq!(
normalize_url("git@github.com:mozilla-firefox/firefox.git"),
expected
);
assert_eq!(
normalize_url("ssh://git@github.com/mozilla-firefox/firefox"),
expected
);
}
fn git(dir: &Path, args: &[&str]) {
let out = Command::new("git")
.current_dir(dir)
.args(args)
.output()
.unwrap();
assert!(
out.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
}
fn paths_in(tmp: &Path) -> Paths {
Paths::discover()
.unwrap()
.with_overrides(Some(tmp.join("data")))
}
fn repo_ref(urls: &[&str]) -> RepoRef {
RepoRef {
urls: urls.iter().map(|s| s.to_string()).collect(),
display_name: "test/repo".into(),
}
}
fn config_with_workdir(workdir: &Path) -> Config {
Config {
workdir: Some(workdir.to_path_buf()),
..Default::default()
}
}
#[test]
fn resolves_a_discovered_repo_without_cloning() {
let tmp = TempDir::new().unwrap();
let owned = tmp.path().join("dev/owned");
std::fs::create_dir_all(&owned).unwrap();
git(&owned, &["init", "-q"]);
git(
&owned,
&["remote", "add", "origin", "git@github.com:o/r.git"],
);
let config = config_with_workdir(&tmp.path().join("dev"));
let mut store = RepoStore::load(&paths_in(tmp.path()), &config).unwrap();
let canon = store
.resolve(&repo_ref(&["https://github.com/o/r"]), OnMissing::Ask)
.unwrap();
assert_eq!(canon.path, owned);
assert_eq!(canon.vcs, VcsKind::Git);
assert!(
!tmp.path().join("data").join("repos").exists(),
"should not have cloned anything"
);
}
#[test]
fn a_cache_miss_triggers_a_rescan_that_picks_up_a_newly_cloned_repo() {
let tmp = TempDir::new().unwrap();
let dev = tmp.path().join("dev");
std::fs::create_dir_all(&dev).unwrap();
let config = config_with_workdir(&dev);
let paths = paths_in(tmp.path());
let mut store = RepoStore::load(&paths, &config).unwrap();
let err = store
.resolve(&repo_ref(&["https://example.com/o/r"]), OnMissing::Ask)
.unwrap_err();
assert!(err.downcast_ref::<NeedsClone>().is_some());
let repo = dev.join("owned");
std::fs::create_dir_all(&repo).unwrap();
git(&repo, &["init", "-q"]);
git(
&repo,
&["remote", "add", "origin", "https://example.com/o/r"],
);
backdate_cache(&paths);
let mut store2 = RepoStore::load(&paths, &config).unwrap();
let canon = store2
.resolve(&repo_ref(&["https://example.com/o/r"]), OnMissing::Ask)
.unwrap();
assert_eq!(canon.path, repo);
}
fn backdate_cache(paths: &Paths) {
let mut cache = WorkdirCache::load(&paths.workdir_cache_file())
.unwrap()
.unwrap();
cache.scanned_at -= RESCAN_THROTTLE * 2;
cache.save(&paths.workdir_cache_file()).unwrap();
}
#[test]
fn a_cached_path_that_no_longer_exists_is_treated_as_a_miss() {
let tmp = TempDir::new().unwrap();
let paths = paths_in(tmp.path());
let dev = tmp.path().join("dev");
std::fs::create_dir_all(&dev).unwrap();
let config = config_with_workdir(&dev);
let cache = WorkdirCache {
workdir: dev.clone(),
scanned_at: chrono::Utc::now(),
repos: vec![DiscoveredRepo {
path: dev.join("gone"),
vcs: VcsKind::Git,
name: "example.com/o/r".into(),
remotes: vec!["example.com/o/r".into()],
}],
};
cache.save(&paths.workdir_cache_file()).unwrap();
let mut store = RepoStore::load(&paths, &config).unwrap();
let err = store
.resolve(&repo_ref(&["https://example.com/o/r"]), OnMissing::Ask)
.unwrap_err();
assert!(err.downcast_ref::<NeedsClone>().is_some());
}
#[test]
fn a_cache_from_a_different_workdir_is_treated_as_a_miss() {
let tmp = TempDir::new().unwrap();
let paths = paths_in(tmp.path());
let old_dev = tmp.path().join("old-dev");
let repo = old_dev.join("owned");
std::fs::create_dir_all(&repo).unwrap();
git(&repo, &["init", "-q"]);
git(
&repo,
&["remote", "add", "origin", "https://example.com/o/r"],
);
let cache = WorkdirCache {
workdir: old_dev,
scanned_at: chrono::Utc::now(),
repos: vec![DiscoveredRepo {
path: repo,
vcs: VcsKind::Git,
name: "example.com/o/r".into(),
remotes: vec!["example.com/o/r".into()],
}],
};
cache.save(&paths.workdir_cache_file()).unwrap();
let config = Config::default();
let mut store = RepoStore::load(&paths, &config).unwrap();
let err = store
.resolve(&repo_ref(&["https://example.com/o/r"]), OnMissing::Ask)
.unwrap_err();
assert!(
err.downcast_ref::<NeedsClone>().is_some(),
"must not serve a repo cached under a different workdir"
);
}
#[test]
fn ask_reports_needs_clone_without_touching_the_filesystem() {
let tmp = TempDir::new().unwrap();
let paths = paths_in(tmp.path());
let config = Config::default();
let mut store = RepoStore::load(&paths, &config).unwrap();
let err = store
.resolve(&repo_ref(&["https://example.com/o/r"]), OnMissing::Ask)
.unwrap_err();
let needs_clone = err.downcast_ref::<NeedsClone>().unwrap();
assert_eq!(needs_clone.url, "https://example.com/o/r");
assert!(!needs_clone.dest.exists());
assert!(!paths.repos_file().exists());
}
#[test]
fn clones_and_reuses_a_tool_managed_repo() {
let tmp = TempDir::new().unwrap();
let upstream = tmp.path().join("upstream");
std::fs::create_dir(&upstream).unwrap();
git(&upstream, &["init", "-q", "-b", "main"]);
git(&upstream, &["config", "user.name", "test"]);
git(&upstream, &["config", "user.email", "test@example.com"]);
git(&upstream, &["commit", "-q", "--allow-empty", "-m", "base"]);
let paths = paths_in(tmp.path());
let config = Config::default();
let url = upstream.to_string_lossy().to_string();
let mut store = RepoStore::load(&paths, &config).unwrap();
let canon1 = store.resolve(&repo_ref(&[&url]), OnMissing::Clone).unwrap();
assert_eq!(canon1.vcs, VcsKind::Git);
assert!(canon1.path.join(".git").exists());
store.save().unwrap();
let mut store2 = RepoStore::load(&paths, &config).unwrap();
let canon2 = store2
.resolve(&repo_ref(&[&url]), OnMissing::Clone)
.unwrap();
assert_eq!(canon2.path, canon1.path);
let canon3 = store2
.resolve(&repo_ref(&[&format!("{url}.git")]), OnMissing::Clone)
.unwrap();
assert_eq!(canon3.path, canon1.path);
}
#[test]
fn resolve_self_heals_an_unregistered_but_already_cloned_directory() {
let tmp = TempDir::new().unwrap();
let upstream = tmp.path().join("upstream");
std::fs::create_dir(&upstream).unwrap();
git(&upstream, &["init", "-q", "-b", "main"]);
git(&upstream, &["config", "user.name", "test"]);
git(&upstream, &["config", "user.email", "test@example.com"]);
git(&upstream, &["commit", "-q", "--allow-empty", "-m", "base"]);
let paths = paths_in(tmp.path());
let config = Config::default();
let url = upstream.to_string_lossy().to_string();
let dest = paths.repo_dir(&normalize_url(&url));
std::fs::create_dir_all(dest.parent().unwrap()).unwrap();
git(tmp.path(), &["clone", "-q", &url, dest.to_str().unwrap()]);
assert!(!paths.repos_file().exists());
let mut store = RepoStore::load(&paths, &config).unwrap();
let canon = store.resolve(&repo_ref(&[&url]), OnMissing::Clone).unwrap();
assert_eq!(canon.path, dest);
assert!(canon.path.join(".git").exists());
}
fn state_using(repo_path: &Path) -> crate::state::State {
let mut state = crate::state::State::default();
let mut entry = entry_using();
entry.stack_id = Some("phab/D1".into());
state.insert(entry);
state.insert_workspace(
"phab/D1".into(),
crate::state::Workspace {
repo_path: repo_path.to_path_buf(),
vcs: VcsKind::Git,
workspace_path: PathBuf::from("/tmp/ws/D1"),
head_id: "abc".into(),
status: crate::state::Status::Ready,
tip: crate::state::ReviewKey::new("phab", "D1"),
version: "1".into(),
},
);
state
}
fn entry_using() -> crate::state::ReviewEntry {
crate::state::ReviewEntry {
key: crate::state::ReviewKey::new("phab", "D1"),
title: "x".into(),
author: "a".into(),
url: "https://example.com/D1".into(),
repo: RepoRef {
urls: vec!["https://example.com/o/r".into()],
display_name: "o/r".into(),
},
kind: crate::source::ReviewKind::Direct,
version: "1".into(),
in_queue: true,
resolved: false,
last_synced: chrono::Utc::now(),
stack_id: None,
ancestors: Vec::new(),
diff_stat: None,
description: None,
}
}
#[test]
fn list_reports_discovered_and_registry_repos_with_workspace_counts() {
let tmp = TempDir::new().unwrap();
let owned = tmp.path().join("dev/owned");
std::fs::create_dir_all(&owned).unwrap();
git(&owned, &["init", "-q"]);
git(
&owned,
&["remote", "add", "origin", "https://example.com/o/r"],
);
let config = config_with_workdir(&tmp.path().join("dev"));
let paths = paths_in(tmp.path());
let mut store = RepoStore::load(&paths, &config).unwrap();
store.rescan().unwrap();
let state = state_using(&owned);
let repos = store.list(&state);
assert_eq!(repos.len(), 1);
assert_eq!(repos[0].kind, RepoKind::Discovered);
assert_eq!(repos[0].workspace_count, 1);
}
#[test]
fn remove_refuses_for_a_discovered_repo() {
let tmp = TempDir::new().unwrap();
let owned = tmp.path().join("dev/owned");
std::fs::create_dir_all(&owned).unwrap();
git(&owned, &["init", "-q"]);
git(
&owned,
&["remote", "add", "origin", "https://example.com/o/r"],
);
let config = config_with_workdir(&tmp.path().join("dev"));
let paths = paths_in(tmp.path());
let mut store = RepoStore::load(&paths, &config).unwrap();
store.rescan().unwrap();
let state = crate::state::State::default();
let err = store.remove("https://example.com/o/r", &state).unwrap_err();
assert!(
err.to_string().contains("workdir"),
"unexpected error: {err}"
);
}
#[test]
fn remove_refuses_while_workspaces_exist() {
let tmp = TempDir::new().unwrap();
let upstream = tmp.path().join("upstream");
std::fs::create_dir(&upstream).unwrap();
git(&upstream, &["init", "-q", "-b", "main"]);
git(&upstream, &["config", "user.name", "test"]);
git(&upstream, &["config", "user.email", "test@example.com"]);
git(&upstream, &["commit", "-q", "--allow-empty", "-m", "base"]);
let url = upstream.to_string_lossy().to_string();
let paths = paths_in(tmp.path());
let config = Config::default();
let mut store = RepoStore::load(&paths, &config).unwrap();
let canon = store.resolve(&repo_ref(&[&url]), OnMissing::Clone).unwrap();
let state = state_using(&canon.path);
let err = store.remove(&url, &state).unwrap_err();
assert!(
err.to_string().contains("still has workspaces"),
"unexpected error: {err}"
);
assert!(canon.path.exists(), "must not delete while still in use");
}
#[test]
fn remove_deletes_an_unused_tool_managed_clone() {
let tmp = TempDir::new().unwrap();
let upstream = tmp.path().join("upstream");
std::fs::create_dir(&upstream).unwrap();
git(&upstream, &["init", "-q", "-b", "main"]);
git(&upstream, &["config", "user.name", "test"]);
git(&upstream, &["config", "user.email", "test@example.com"]);
git(&upstream, &["commit", "-q", "--allow-empty", "-m", "base"]);
let url = upstream.to_string_lossy().to_string();
let paths = paths_in(tmp.path());
let config = Config::default();
let mut store = RepoStore::load(&paths, &config).unwrap();
let canon = store.resolve(&repo_ref(&[&url]), OnMissing::Clone).unwrap();
store.save().unwrap();
let state = crate::state::State::default();
store.remove(&url, &state).unwrap();
store.save().unwrap();
assert!(!canon.path.exists());
let mut reloaded = RepoStore::load(&paths, &config).unwrap();
let canon2 = reloaded
.resolve(&repo_ref(&[&url]), OnMissing::Clone)
.unwrap();
assert!(canon2.path.exists());
}
}