use std::process::Output;
use std::time::Duration;
use tokio::process::Command;
use super::aws::{self, AwsTempCreds};
use super::types::{CredentialKind, Credentials, LsRemoteResult, RemoteHead};
use crate::call::not_run;
use crate::clone::https::HttpsAccess;
use crate::clone::reason::hide;
use crate::clone::ssh::{drop_scp_scheme, SshKey, ONLY_SSH};
use crate::clone::{has_http_scheme, redirect_config};
use crate::errors::{
CodecommitLsRemoteError, GitCallError, HttpsAccessError, HttpsLsRemoteError, LsRemoteError,
};
const LS_REMOTE_TIMEOUT_SECS: u64 = 20;
const LS_REMOTE_TIMEOUT_MESSAGE: &str = "git ls-remote time out";
struct AwsRole {
arn: String,
external_id: String,
via: Option<String>,
}
trait Answer {
const SYMREF: bool;
fn timed_out() -> Self;
fn read(output: &Output, named: &str, secrets: &[&str]) -> Self;
}
impl Answer for LsRemoteResult {
const SYMREF: bool = false;
fn timed_out() -> Self {
Self {
commit: None,
error: Some(LS_REMOTE_TIMEOUT_MESSAGE.to_owned()),
}
}
fn read(output: &Output, branch: &str, secrets: &[&str]) -> Self {
parse_ls_remote_output(output, branch, secrets)
}
}
impl Answer for RemoteHead {
const SYMREF: bool = true;
fn timed_out() -> Self {
Self {
branch: None,
error: Some(LS_REMOTE_TIMEOUT_MESSAGE.to_owned()),
}
}
fn read(output: &Output, _named: &str, secrets: &[&str]) -> Self {
parse_symref_output(output, secrets)
}
}
fn branch_refs(branch: &str) -> Vec<String> {
if branch == "HEAD" {
return vec!["HEAD".to_owned()];
}
vec![
format!("refs/heads/{branch}"),
format!("refs/tags/{branch}"),
]
}
fn ls_remote_args(url: &str, pattern: &str, symref: bool) -> Vec<String> {
let mut args = vec!["ls-remote".to_owned()];
if symref {
args.push("--symref".to_owned());
}
args.extend(["--".to_owned(), url.to_owned(), pattern.to_owned()]);
args
}
async fn run_ls_remote<A: Answer>(
command: &mut Command,
named: &str,
secrets: &[&str],
) -> Result<A, GitCallError> {
let Ok(output) = tokio::time::timeout(
Duration::from_secs(LS_REMOTE_TIMEOUT_SECS),
command.kill_on_drop(true).output(),
)
.await
else {
return Ok(A::timed_out());
};
let output = output.map_err(|source| not_run("ls-remote", source))?;
Ok(A::read(&output, named, secrets))
}
pub async fn ls_remote(
url: &str,
branch: &str,
credentials: Option<&Credentials>,
follow_redirects: bool,
) -> anyhow::Result<LsRemoteResult> {
ask(url, branch, credentials, follow_redirects).await
}
pub async fn ssh_ls_remote(
url: &str,
branch: &str,
key: &str,
) -> Result<LsRemoteResult, LsRemoteError> {
ls_remote_ssh(url, branch, key).await
}
pub async fn codecommit_ls_remote(
url: &str,
branch: &str,
arn: &str,
external_id: &str,
follow_redirects: bool,
) -> Result<LsRemoteResult, CodecommitLsRemoteError> {
let role = AwsRole {
arn: arn.to_owned(),
external_id: external_id.to_owned(),
via: None,
};
ls_remote_codecommit(url, branch, &role, follow_redirects).await
}
pub async fn https_ls_remote(
url: &str,
branch: &str,
credential: Option<&CredentialKind>,
follow_redirects: bool,
) -> Result<LsRemoteResult, HttpsLsRemoteError> {
ls_remote_https(url, branch, credential, follow_redirects).await
}
pub async fn remote_head(
url: &str,
credentials: Option<&Credentials>,
follow_redirects: bool,
) -> anyhow::Result<RemoteHead> {
ask(url, "HEAD", credentials, follow_redirects).await
}
async fn ask<A: Answer>(
url: &str,
pattern: &str,
credentials: Option<&Credentials>,
follow_redirects: bool,
) -> anyhow::Result<A> {
match credentials.map(|c| &c.kind) {
Some(CredentialKind::Ssh { key }) => Ok(ls_remote_ssh(url, pattern, key).await?),
Some(CredentialKind::AwsRole {
arn,
external_id,
via,
}) => {
let role = AwsRole {
arn: arn.clone(),
external_id: external_id.clone(),
via: via.clone(),
};
Ok(ls_remote_codecommit(url, pattern, &role, follow_redirects).await?)
}
kind => Ok(ls_remote_https(url, pattern, kind, follow_redirects).await?),
}
}
async fn ls_remote_ssh<A: Answer>(
url: &str,
pattern: &str,
key_b64: &str,
) -> Result<A, LsRemoteError> {
let key = SshKey::write(key_b64)?;
let args = ls_remote_args(drop_scp_scheme(url), pattern, A::SYMREF);
let mut command = Command::new("git");
command
.args(&args)
.env("GIT_SSH_COMMAND", &key.command)
.env("GIT_ALLOW_PROTOCOL", ONLY_SSH);
Ok(run_ls_remote(&mut command, pattern, &[]).await?)
}
async fn ls_remote_https<A: Answer>(
url: &str,
pattern: &str,
credential: Option<&CredentialKind>,
follow_redirects: bool,
) -> Result<A, HttpsLsRemoteError> {
if credential.is_none() && !has_http_scheme(url) {
return Err(HttpsAccessError::NotHttp.into());
}
let access = HttpsAccess::new(url, credential)?;
let mut args = vec!["-c".to_owned(), "http.sslVerify=false".to_owned()];
args.extend(redirect_config(follow_redirects));
args.extend(ls_remote_args(access.url(), pattern, A::SYMREF));
let mut command = access.git();
command.args(&args);
Ok(run_ls_remote(&mut command, pattern, access.secret().as_slice()).await?)
}
async fn ls_remote_codecommit<A: Answer>(
url: &str,
pattern: &str,
role: &AwsRole,
follow_redirects: bool,
) -> Result<A, CodecommitLsRemoteError> {
let creds = aws::assume_role(&role.arn, &role.external_id, url, role.via.as_deref()).await?;
Ok(ls_remote_with(url, pattern, &creds, follow_redirects).await?)
}
async fn ls_remote_with<A: Answer>(
url: &str,
pattern: &str,
creds: &AwsTempCreds,
follow_redirects: bool,
) -> Result<A, GitCallError> {
let args = codecommit_ls_remote_args(url, pattern, A::SYMREF, follow_redirects);
let mut command = Command::new("git");
command
.args(&args)
.env("AWS_ACCESS_KEY_ID", &creds.access_key)
.env("AWS_SECRET_ACCESS_KEY", &creds.secret_key)
.env("AWS_SESSION_TOKEN", &creds.session_token)
.env("AWS_DEFAULT_REGION", &creds.region);
let secrets = [creds.secret_key.as_str(), creds.session_token.as_str()];
run_ls_remote(&mut command, pattern, &secrets).await
}
fn codecommit_ls_remote_args(
url: &str,
pattern: &str,
symref: bool,
follow_redirects: bool,
) -> Vec<String> {
let mut args = redirect_config(follow_redirects).to_vec();
args.extend(ls_remote_args(url, pattern, symref));
args
}
fn parse_ls_remote_output(output: &Output, branch: &str, secrets: &[&str]) -> LsRemoteResult {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return LsRemoteResult {
commit: None,
error: Some(hide(&stderr, secrets)),
};
}
let stdout = String::from_utf8_lossy(&output.stdout);
let commit_of = |wanted: &str| {
stdout.lines().find_map(|line| {
let (commit, name) = line.split_once('\t')?;
(name == wanted).then_some(commit)
})
};
let commit = branch_refs(branch)
.iter()
.find_map(|wanted| commit_of(wanted))
.unwrap_or_default()
.to_owned();
LsRemoteResult {
commit: Some(commit),
error: None,
}
}
fn parse_symref_output(output: &Output, secrets: &[&str]) -> RemoteHead {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return RemoteHead {
branch: None,
error: Some(hide(&stderr, secrets)),
};
}
let stdout = String::from_utf8_lossy(&output.stdout);
RemoteHead {
branch: stdout.lines().find_map(head_branch).map(str::to_owned),
error: None,
}
}
fn head_branch(line: &str) -> Option<&str> {
let (target, name) = line.strip_prefix("ref: ")?.split_once('\t')?;
if name != "HEAD" {
return None;
}
target
.strip_prefix("refs/heads/")
.filter(|branch| !branch.is_empty())
}
#[cfg(test)]
mod tests {
use std::io::{BufRead, BufReader, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::{Arc, Mutex};
use base64::Engine;
use super::{
codecommit_ls_remote_args, https_ls_remote, ls_remote, ls_remote_args, ls_remote_with,
parse_ls_remote_output, parse_symref_output, redirect_config, remote_head, run_ls_remote,
ssh_ls_remote, AwsTempCreds,
};
use crate::clone::https::HttpsAccess;
use crate::errors::{GitCallError, HttpsAccessError, HttpsLsRemoteError, LsRemoteError};
use crate::types::{CredentialKind, Credentials, LsRemoteResult};
fn read_request(stream: &TcpStream) -> (String, Option<String>) {
let mut path = String::new();
let mut authorization = None;
let mut reader = BufReader::new(stream);
let mut line = String::new();
while reader.read_line(&mut line).is_ok_and(|read| read > 0) {
let header = line.trim_end();
if header.is_empty() {
break;
}
if path.is_empty() {
header
.split(' ')
.nth(1)
.unwrap_or_default()
.clone_into(&mut path);
}
if let Some(value) = header.strip_prefix("Authorization: ") {
authorization = Some(value.to_owned());
}
line.clear();
}
(path, authorization)
}
fn answer(redirect_to: Option<&str>, path: &str) -> String {
redirect_to.map_or_else(
|| {
"HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"x\"\r\n\
Content-Length: 0\r\nConnection: close\r\n\r\n"
.to_owned()
},
|host| {
format!(
"HTTP/1.1 302 Found\r\nLocation: {host}{path}\r\n\
Content-Length: 0\r\nConnection: close\r\n\r\n"
)
},
)
}
fn serve(redirect_to: Option<String>) -> (String, Arc<Mutex<Vec<Option<String>>>>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
let port = listener.local_addr().expect("addr").port();
let seen = Arc::new(Mutex::new(Vec::new()));
let record = Arc::clone(&seen);
std::thread::spawn(move || {
for stream in listener.incoming().flatten() {
let (path, authorization) = read_request(&stream);
record.lock().expect("lock").push(authorization);
let answer = answer(redirect_to.as_deref(), &path);
let _ = (&stream).write_all(answer.as_bytes());
}
});
(format!("http://127.0.0.1:{port}"), seen)
}
fn token(token: &str) -> Credentials {
Credentials {
kind: CredentialKind::Token {
token: token.to_owned(),
is_pat: false,
oauth_type: String::new(),
},
}
}
fn pat(token: &str) -> Credentials {
Credentials {
kind: CredentialKind::Token {
token: token.to_owned(),
is_pat: true,
oauth_type: String::new(),
},
}
}
fn output(code: i32, stdout: &str, stderr: &str) -> std::process::Output {
use std::os::unix::process::ExitStatusExt;
std::process::Output {
status: std::process::ExitStatus::from_raw(code << 8),
stdout: stdout.as_bytes().to_vec(),
stderr: stderr.as_bytes().to_vec(),
}
}
#[test]
fn redirects_are_forbidden_explicitly_and_not_left_to_the_git_default() {
assert_eq!(
redirect_config(false),
["-c".to_owned(), "http.followRedirects=false".to_owned()]
);
assert_eq!(
redirect_config(true),
["-c".to_owned(), "http.followRedirects=true".to_owned()]
);
}
#[test]
fn a_branch_missing_from_the_remote_is_an_empty_commit_and_not_a_missing_one() {
let result = parse_ls_remote_output(&output(0, "", ""), "main", &[]);
assert_eq!(result.commit, Some(String::new()));
assert!(result.error.is_none());
}
#[test]
fn a_branch_is_read_by_its_full_name_and_before_a_tag_of_the_same_name() {
for (said, commit) in [
(
"aaa\trefs/heads/feature/main\nbbb\trefs/heads/main\n",
"bbb",
),
("ttt\trefs/tags/main\nbbb\trefs/heads/main\n", "bbb"),
("ttt\trefs/tags/main\n", "ttt"),
("aaa\trefs/heads/feature/main\n", ""),
] {
let result = parse_ls_remote_output(&output(0, said, ""), "main", &[]);
assert_eq!(result.commit.as_deref(), Some(commit), "{said:?}");
}
}
#[test]
fn head_is_read_only_from_the_line_git_lists_as_head() {
for (said, commit) in [
("aaa\tHEAD\nbbb\trefs/heads/HEAD\n", "aaa"),
("bbb\trefs/heads/HEAD\nccc\trefs/remotes/origin/HEAD\n", ""),
] {
let result = parse_ls_remote_output(&output(0, said, ""), "HEAD", &[]);
assert_eq!(result.commit.as_deref(), Some(commit), "{said:?}");
}
}
#[test]
fn a_failure_reports_the_stderr_and_no_commit() {
let result =
parse_ls_remote_output(&output(128, "", "fatal: repository not found"), "main", &[]);
assert!(result.commit.is_none());
assert_eq!(result.error.as_deref(), Some("fatal: repository not found"));
}
#[tokio::test]
async fn a_token_reaches_its_host_and_stays_out_of_the_error() {
for (credentials, login) in [
(token("Zm9v+YmFy"), "Zm9v+YmFy:"),
(pat("Zm9v+YmFy"), "pat:Zm9v+YmFy"),
] {
let (host, seen) = serve(None);
let result = ls_remote(
&format!("{host}/org/repo.git"),
"main",
Some(&credentials),
false,
)
.await
.unwrap();
let sent = base64::engine::general_purpose::STANDARD.encode(login);
assert!(seen
.lock()
.unwrap()
.contains(&Some(format!("Basic {sent}"))));
let error = result.error.unwrap();
assert!(error.contains("Authentication failed"), "{error}");
assert!(!error.contains("Zm9v"), "{error}");
}
}
#[tokio::test]
async fn a_redirect_to_another_host_carries_no_credential() {
for credentials in [token("tok123"), pat("tok123")] {
let (other, other_seen) = serve(None);
let other = other.replace("127.0.0.1", "localhost");
let (host, _) = serve(Some(other));
let result = ls_remote(
&format!("{host}/org/repo.git"),
"main",
Some(&credentials),
true,
)
.await
.unwrap();
assert!(result.error.is_some());
let other_seen = other_seen.lock().unwrap().clone();
assert!(!other_seen.is_empty());
assert!(other_seen.iter().all(Option::is_none), "{other_seen:?}");
}
}
#[tokio::test]
async fn a_helper_the_caller_configured_never_sees_the_credential() {
let (host, _) = serve(None);
let calls = tempfile::NamedTempFile::new().unwrap();
let access =
HttpsAccess::new(&format!("{host}/org/repo.git"), Some(&token("tok123").kind)).unwrap();
let mut command = access.git();
command
.env(
"GIT_CONFIG_PARAMETERS",
format!(
"'credential.helper'='!f() {{ echo $1 >> {}; }}; f'",
calls.path().display()
),
)
.args(["ls-remote", "--", access.url(), "main"]);
run_ls_remote::<LsRemoteResult>(&mut command, "main", access.secret().as_slice())
.await
.unwrap();
assert_eq!(std::fs::read_to_string(calls.path()).unwrap(), "");
}
#[test]
fn a_failure_hides_the_credential_a_server_echoes() {
let said = "remote: token tok123 is not valid\nfatal: Authentication failed\n";
let result = parse_ls_remote_output(&output(128, "", said), "main", &["tok123"]);
assert_eq!(
result.error.as_deref(),
Some("remote: token [REDACTED] is not valid\nfatal: Authentication failed\n")
);
}
#[test]
fn a_codecommit_url_lets_git_check_the_certificate() {
for symref in [false, true] {
let args =
codecommit_ls_remote_args("codecommit::us-east-1://repo", "main", symref, false);
assert!(!args.contains(&"http.sslVerify=false".to_owned()));
}
}
#[tokio::test]
async fn a_codecommit_ls_remote_keeps_the_session_out_of_what_git_says() {
let tmp = tempfile::tempdir().unwrap();
let url = format!("{}/a-secret-key/a-token", tmp.path().display());
let creds = AwsTempCreds {
access_key: "an-access-key".to_owned(),
secret_key: "a-secret-key".to_owned(),
session_token: "a-token".to_owned(),
region: "us-east-1".to_owned(),
};
let answer: LsRemoteResult = ls_remote_with(&url, "main", &creds, false)
.await
.expect("git ran");
let said = answer.error.expect("git refused");
assert!(said.contains("[REDACTED]/[REDACTED]"), "{said}");
}
#[test]
fn symbolic_refs_are_asked_for_only_when_wanted_and_before_the_url() {
assert_eq!(
ls_remote_args("https://h/r", "HEAD", true),
["ls-remote", "--symref", "--", "https://h/r", "HEAD"]
);
assert_eq!(
ls_remote_args("https://h/r", "main", false),
["ls-remote", "--", "https://h/r", "main"]
);
}
fn repository(branch: &str, commits: usize) -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
let git = |args: &[&str]| {
let status = std::process::Command::new("git")
.arg("-C")
.arg(dir.path())
.args(["-c", "user.name=t", "-c", "user.email=t@example.com"])
.args(args)
.output()
.unwrap()
.status;
assert!(status.success(), "git {args:?}");
};
git(&["init", "--quiet", "--initial-branch", branch]);
for _ in 0..commits {
git(&["commit", "--quiet", "--allow-empty", "--message", "c"]);
}
dir
}
fn file_answer(git_dir: &std::path::Path, path: &str) -> Vec<u8> {
let file = path.split('?').next().unwrap_or_default();
let Ok(body) = std::fs::read(git_dir.join(file.trim_start_matches('/'))) else {
return b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
.to_vec();
};
let head = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
[head.into_bytes(), body].concat()
}
fn serve_repository(repository: &std::path::Path) -> String {
let git_dir = repository.join(".git");
let updated = std::process::Command::new("git")
.arg("-C")
.arg(&git_dir)
.arg("update-server-info")
.status()
.unwrap();
assert!(updated.success());
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
let port = listener.local_addr().expect("addr").port();
std::thread::spawn(move || {
for stream in listener.incoming().flatten() {
let (path, _) = read_request(&stream);
let _ = (&stream).write_all(&file_answer(&git_dir, &path));
}
});
format!("http://127.0.0.1:{port}/")
}
#[test]
fn the_branch_head_names_is_read_without_its_prefix() {
let result = parse_symref_output(
&output(0, "ref: refs/heads/release/2.0\tHEAD\nabc123\tHEAD\n", ""),
&[],
);
assert_eq!(result.branch.as_deref(), Some("release/2.0"));
assert!(result.error.is_none());
}
#[test]
fn a_head_that_names_no_branch_is_no_branch_and_no_error() {
for said in [
"",
"abc123\tHEAD\n",
"ref: refs/tags/v1\tHEAD\n",
"ref: refs/heads/\tHEAD\n",
"ref: refs/heads/main\trefs/remotes/origin/HEAD\n",
] {
let result = parse_symref_output(&output(0, said, ""), &[]);
assert!(result.branch.is_none(), "{said:?}");
assert!(result.error.is_none(), "{said:?}");
}
}
#[test]
fn a_failed_head_read_reports_the_stderr_with_the_credential_hidden() {
let said = "remote: token tok123 is not valid\nfatal: Authentication failed\n";
let result = parse_symref_output(&output(128, "", said), &["tok123"]);
assert!(result.branch.is_none());
assert_eq!(
result.error.as_deref(),
Some("remote: token [REDACTED] is not valid\nfatal: Authentication failed\n")
);
}
#[tokio::test]
async fn a_remote_answers_the_branch_its_head_names() {
let remote = repository("trunk", 1);
let url = serve_repository(remote.path());
let result = remote_head(&url, None, false).await.unwrap();
assert_eq!(result.branch.as_deref(), Some("trunk"));
assert!(result.error.is_none());
}
#[tokio::test]
async fn an_empty_remote_names_no_branch() {
let remote = repository("trunk", 0);
let url = serve_repository(remote.path());
let result = remote_head(&url, None, false).await.unwrap();
assert!(result.branch.is_none());
assert!(result.error.is_none());
}
#[tokio::test]
async fn a_branch_is_not_found_by_a_ref_that_only_ends_in_its_name() {
let remote = repository("trunk", 1);
let moved = std::process::Command::new("git")
.arg("-C")
.arg(remote.path())
.args(["update-ref", "refs/heads/feature/dev", "trunk"])
.status()
.unwrap();
assert!(moved.success());
let url = serve_repository(remote.path());
let found = ls_remote(&url, "trunk", None, false).await.unwrap();
let missing = ls_remote(&url, "dev", None, false).await.unwrap();
assert_eq!(found.commit.unwrap().len(), 40);
assert_eq!(missing.commit.as_deref(), Some(""));
}
#[tokio::test]
async fn without_a_credential_a_url_that_is_not_http_never_runs_git() {
let remote = repository("trunk", 1);
let path = remote.path().to_str().unwrap();
for url in [
path,
"httpx://127.0.0.1/repo.git",
"http:127.0.0.1/repo.git",
] {
let answer = https_ls_remote(url, "trunk", None, false).await;
assert!(
matches!(
answer,
Err(HttpsLsRemoteError::Access(HttpsAccessError::NotHttp))
),
"{url}: {answer:?}"
);
}
}
#[tokio::test]
async fn an_ssh_ls_remote_with_a_nul_in_the_branch_never_runs_git() {
let key = base64::engine::general_purpose::STANDARD.encode("not a real key");
let nul = ssh_ls_remote("git@example.com:org/repo.git", "ma\0in", &key).await;
assert!(matches!(
nul,
Err(LsRemoteError::Git(GitCallError::NullByte {
command: "ls-remote"
}))
));
}
#[tokio::test]
async fn a_key_reads_no_head_from_a_url_that_is_not_ssh() {
let remote = repository("trunk", 1);
let url = remote.path().to_str().unwrap();
let key = Credentials {
kind: CredentialKind::Ssh {
key: base64::engine::general_purpose::STANDARD.encode("not a real key"),
},
};
let result = remote_head(url, Some(&key), false).await.unwrap();
assert!(result.branch.is_none());
assert!(result.error.is_some());
}
#[tokio::test]
async fn a_head_read_takes_the_token_to_its_host_and_out_of_the_error() {
for (credentials, login) in [
(token("Zm9v+YmFy"), "Zm9v+YmFy:"),
(pat("Zm9v+YmFy"), "pat:Zm9v+YmFy"),
] {
let (host, seen) = serve(None);
let result = remote_head(&format!("{host}/org/repo.git"), Some(&credentials), false)
.await
.unwrap();
let sent = base64::engine::general_purpose::STANDARD.encode(login);
assert!(seen
.lock()
.unwrap()
.contains(&Some(format!("Basic {sent}"))));
assert!(result.branch.is_none());
let error = result.error.unwrap();
assert!(error.contains("Authentication failed"), "{error}");
assert!(!error.contains("Zm9v"), "{error}");
}
}
#[tokio::test]
async fn a_head_read_redirected_to_another_host_carries_no_credential() {
for credentials in [token("tok123"), pat("tok123")] {
let (other, other_seen) = serve(None);
let other = other.replace("127.0.0.1", "localhost");
let (host, _) = serve(Some(other));
let result = remote_head(&format!("{host}/org/repo.git"), Some(&credentials), true)
.await
.unwrap();
assert!(result.error.is_some());
let other_seen = other_seen.lock().unwrap().clone();
assert!(!other_seen.is_empty());
assert!(other_seen.iter().all(Option::is_none), "{other_seen:?}");
}
}
}