#![cfg(feature = "aws-profile")]
use std::io::{BufRead, BufReader, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::sync::{Arc, Mutex};
const BUCKET: &str = "bucket";
const KEY: &str = "karate_edges.csv";
fn karate_edges() -> Vec<u8> {
let path = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.join("python/ursa/datasets/data/karate_edges.csv");
std::fs::read(path).unwrap()
}
struct FakeS3 {
endpoint: String,
signatures: Arc<Mutex<Vec<String>>>,
}
impl FakeS3 {
fn start(key_id: &'static str) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let signatures = Arc::new(Mutex::new(Vec::new()));
let seen = Arc::clone(&signatures);
let body = karate_edges();
std::thread::spawn(move || {
for stream in listener.incoming().flatten() {
serve(stream, key_id, &body, &seen);
}
});
Self {
endpoint,
signatures,
}
}
fn signatures(&self) -> Vec<String> {
self.signatures.lock().unwrap().clone()
}
}
fn serve(mut stream: TcpStream, key_id: &str, body: &[u8], seen: &Mutex<Vec<String>>) {
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut request_line = String::new();
if reader.read_line(&mut request_line).is_err() {
return;
}
let mut authorization = String::new();
let mut range: Option<(usize, usize)> = None;
loop {
let mut line = String::new();
if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" {
break;
}
let (name, value) = line.split_once(':').unwrap_or((&line, ""));
let value = value.trim();
match name.to_ascii_lowercase().as_str() {
"authorization" => authorization = value.to_string(),
"range" => {
range = value
.strip_prefix("bytes=")
.and_then(|r| r.split_once('-'))
.map(|(a, b)| {
let start = a.parse().unwrap();
let end = b.parse::<usize>().map_or(body.len(), |e| e + 1);
(start, end.min(body.len()))
})
}
_ => {}
}
}
seen.lock().unwrap().push(authorization.clone());
let mut parts = request_line.split_whitespace();
let method = parts.next().unwrap_or("");
let target = parts.next().unwrap_or("");
let (path, query) = target.split_once('?').unwrap_or((target, ""));
let head = method == "HEAD";
if !authorization.contains(&format!("Credential={key_id}/")) {
return xml(
&mut stream,
"403 Forbidden",
"<Error><Code>InvalidAccessKeyId</Code><Message>not the expected key</Message>\
</Error>"
.to_string(),
);
}
let object = format!("/{BUCKET}/{KEY}");
let etag = "\"0123456789abcdef\"".to_string();
if path.trim_end_matches('/') == format!("/{BUCKET}") && query.contains("list-type=2") {
let prefix = query
.split('&')
.find_map(|kv| kv.strip_prefix("prefix="))
.unwrap_or("");
let contents = if KEY.starts_with(prefix) {
format!(
"<Contents><Key>{KEY}</Key><LastModified>2024-01-01T00:00:00.000Z</LastModified>\
<ETag>{etag}</ETag><Size>{}</Size><StorageClass>STANDARD</StorageClass>\
</Contents>",
body.len()
)
} else {
String::new()
};
return xml(
&mut stream,
"200 OK",
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListBucketResult><Name>{BUCKET}</Name>\
<Prefix>{prefix}</Prefix><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>\
{contents}</ListBucketResult>"
),
);
}
if path == object {
let meta = |len: usize| {
vec![
("Content-Length", len.to_string()),
("Last-Modified", "Mon, 01 Jan 2024 00:00:00 GMT".to_string()),
("ETag", etag.clone()),
]
};
return match range {
Some((start, end)) if method == "GET" => {
let mut headers = meta(end - start);
headers.push((
"Content-Range",
format!("bytes {start}-{}/{}", end - 1, body.len()),
));
reply(
&mut stream,
head,
"206 Partial Content",
&headers,
&body[start..end],
)
}
_ => reply(&mut stream, head, "200 OK", &meta(body.len()), body),
};
}
xml(
&mut stream,
"404 Not Found",
"<Error><Code>NoSuchKey</Code><Message>no such key</Message></Error>".to_string(),
)
}
fn reply(
stream: &mut TcpStream,
head: bool,
status: &str,
headers: &[(&str, String)],
payload: &[u8],
) {
let mut text = format!("HTTP/1.1 {status}\r\nConnection: close\r\n");
for (name, value) in headers {
text.push_str(&format!("{name}: {value}\r\n"));
}
text.push_str("\r\n");
let mut out = text.into_bytes();
if !head {
out.extend_from_slice(payload);
}
let _ = stream.write_all(&out);
}
fn xml(stream: &mut TcpStream, status: &str, text: String) {
let headers = [
("Content-Type", "application/xml".to_string()),
("Content-Length", text.len().to_string()),
];
reply(stream, false, status, &headers, text.as_bytes());
}
fn aws_home(credentials: &str, config: &str) -> tempfile::TempDir {
let home = tempfile::tempdir().unwrap();
let aws = home.path().join(".aws");
std::fs::create_dir(&aws).unwrap();
std::fs::write(aws.join("credentials"), credentials).unwrap();
std::fs::write(aws.join("config"), config).unwrap();
home
}
fn ursa(home: &Path, s3: &FakeS3, env: &[(&str, &str)], sql: &str) -> Output {
let mut command = Command::new(env!("CARGO_BIN_EXE_ursa"));
command
.args(["--format", "csv", "-c", sql])
.env_clear()
.env("HOME", home)
.env("AWS_ENDPOINT", &s3.endpoint)
.env("AWS_ALLOW_HTTP", "true")
.stdin(Stdio::null());
for (k, v) in env {
command.env(k, v);
}
command.output().expect("ursa runs")
}
const COUNT: &str = "SELECT count(*) AS edges FROM 's3://bucket/karate_edges.csv'";
fn counted(output: Output) -> String {
assert!(
output.status.success(),
"ursa failed: {}",
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8(output.stdout).unwrap()
}
#[test]
fn a_named_profile_signs_the_requests() {
let s3 = FakeS3::start("AKIDPROFILE");
let home = aws_home(
"[default]\naws_access_key_id = AKIDDEFAULT\naws_secret_access_key = default-secret\n\n\
[analytics]\naws_access_key_id = AKIDPROFILE\naws_secret_access_key = profile-secret\n",
"[profile analytics]\nregion = eu-west-3\n",
);
let out = ursa(home.path(), &s3, &[("AWS_PROFILE", "analytics")], COUNT);
assert_eq!(counted(out), "edges\n78\n");
let signatures = s3.signatures();
assert!(!signatures.is_empty());
for s in signatures {
assert!(
s.contains("Credential=AKIDPROFILE/") && s.contains("/eu-west-3/s3/"),
"{s}"
);
}
}
#[test]
fn the_default_profile_is_read_without_aws_profile() {
let s3 = FakeS3::start("AKIDDEFAULT");
let home = aws_home(
"[default]\naws_access_key_id = AKIDDEFAULT\naws_secret_access_key = default-secret\n",
"",
);
assert_eq!(counted(ursa(home.path(), &s3, &[], COUNT)), "edges\n78\n");
}
#[test]
fn files_named_by_the_environment_are_read() {
let s3 = FakeS3::start("AKIDELSEWHERE");
let home = aws_home("", "");
let dir = tempfile::tempdir().unwrap();
let credentials: PathBuf = dir.path().join("creds");
std::fs::write(
&credentials,
"[ci]\naws_access_key_id = AKIDELSEWHERE\naws_secret_access_key = s\n",
)
.unwrap();
let out = ursa(
home.path(),
&s3,
&[
("AWS_PROFILE", "ci"),
("AWS_SHARED_CREDENTIALS_FILE", credentials.to_str().unwrap()),
],
COUNT,
);
assert_eq!(counted(out), "edges\n78\n");
}
#[test]
fn environment_keys_win_over_the_profile() {
let s3 = FakeS3::start("AKIDENV");
let home = aws_home(
"[default]\naws_access_key_id = AKIDDEFAULT\naws_secret_access_key = default-secret\n",
"",
);
let out = ursa(
home.path(),
&s3,
&[
("AWS_ACCESS_KEY_ID", "AKIDENV"),
("AWS_SECRET_ACCESS_KEY", "env-secret"),
],
COUNT,
);
assert_eq!(counted(out), "edges\n78\n");
}
#[cfg(unix)]
#[test]
fn a_credential_process_is_run() {
let s3 = FakeS3::start("AKIDPROCESS");
let dir = tempfile::tempdir().unwrap();
let json = dir.path().join("credentials.json");
std::fs::write(
&json,
r#"{"Version": 1, "AccessKeyId": "AKIDPROCESS", "SecretAccessKey": "process-secret"}"#,
)
.unwrap();
let home = aws_home(
"",
&format!(
"[profile tool]\ncredential_process = cat {}\n",
json.display()
),
);
let out = ursa(
home.path(),
&s3,
&[("AWS_PROFILE", "tool"), ("PATH", "/usr/bin:/bin")],
COUNT,
);
assert_eq!(counted(out), "edges\n78\n");
}
#[test]
fn a_named_profile_that_gives_no_credentials_is_an_error() {
let s3 = FakeS3::start("AKIDDEFAULT");
let home = aws_home(
"[default]\naws_access_key_id = AKIDDEFAULT\naws_secret_access_key = s\n",
"",
);
let out = ursa(home.path(), &s3, &[("AWS_PROFILE", "nowhere")], COUNT);
assert!(!out.status.success());
let stderr = String::from_utf8(out.stderr).unwrap();
assert!(
stderr
.contains("cannot connect to s3://bucket: AWS profile 'nowhere' gives no credentials"),
"{stderr}"
);
assert!(s3.signatures().is_empty());
}
#[test]
fn a_wrong_key_is_refused_by_the_bucket() {
let s3 = FakeS3::start("AKIDEXPECTED");
let home = aws_home(
"[default]\naws_access_key_id = AKIDOTHER\naws_secret_access_key = s\n",
"",
);
let out = ursa(home.path(), &s3, &[], COUNT);
assert!(!out.status.success());
}