use std::collections::HashMap;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use http_body_util::{BodyExt, Empty, Full, Limited, StreamBody};
use hyper::body::{Bytes, Frame, Incoming};
use hyper::header::{CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, HOST, HeaderName, HeaderValue};
use hyper::{HeaderMap, Method, Request, Response, StatusCode};
use serde::Deserialize;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tracing::debug;
mod save;
pub(super) use save::Save;
use save::{Finished, Part};
use super::serve::{Body, Purpose, Receiving, drain, receive, status};
use super::{ActionsCache, Inner, Runner, State, Upload, Writes, charge, valid_segment};
use crate::sync::lock;
pub(super) const TWIRP: &str = "twirp/";
pub(super) const BLOBS: &str = "runnerdcache/";
const CACHE_SERVICE: &str = "twirp/github.actions.results.api.v1.CacheService/";
const UPLOADS: &str = "runnerdcache/upload/";
const DOWNLOADS: &str = "runnerdcache/download/";
const RESULTS_ORIGIN: &str = "https://results-receiver.actions.githubusercontent.com";
const TOOL: &str = "actions";
const END: &str = "~";
const SEGMENT: usize = 128;
const MAX_KEY: usize = 512;
const MAX_REQUEST: usize = 64 << 10;
const MAX_BLOCK_LIST: usize = 4 << 20;
const MAX_PASSED: usize = 1 << 20;
const PASS_TIMEOUT: Duration = Duration::from_secs(60);
pub(super) const MAX_SAVES: usize = 1_000;
const MAX_BLOCKS: usize = 50_000;
const MAX_FOUND: usize = 10_000;
const DEFAULT_HOST: &str = "127.0.0.1:41772";
const REQUEST_ID: HeaderName = HeaderName::from_static("x-ms-request-id");
const MS_RANGE: HeaderName = HeaderName::from_static("x-ms-range");
#[derive(Debug)]
pub(super) struct Results {
origin: Mutex<String>,
client: OnceLock<Option<reqwest::Client>>,
}
impl Results {
pub(super) fn new() -> Results {
Results {
origin: Mutex::new(RESULTS_ORIGIN.to_owned()),
client: OnceLock::new(),
}
}
#[cfg(test)]
pub(super) fn set_origin(&self, origin: &str) {
*lock(&self.origin) = origin.to_owned();
}
fn client(&self) -> Option<&reqwest::Client> {
self.client
.get_or_init(|| {
crate::github::install_crypto_provider();
reqwest::Client::builder()
.timeout(PASS_TIMEOUT)
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| debug!("cannot make the results service's client: {error}"))
.ok()
})
.as_ref()
}
}
pub(super) async fn handle(
inner: &Arc<Inner>,
runner: &str,
request: Request<Incoming>,
) -> Response<Body> {
let served = lock(&inner.state)
.runners
.get(runner)
.is_some_and(|open| open.actions == ActionsCache::Served);
let path = request.uri().path().trim_start_matches('/').to_owned();
if let Some(method) = path.strip_prefix(CACHE_SERVICE) {
if request.method() != Method::POST {
return twirp_error(StatusCode::NOT_FOUND, "bad_route");
}
let host = host(request.headers());
let Some(body) = read(request.into_body(), MAX_REQUEST).await else {
return twirp_error(StatusCode::BAD_REQUEST, "malformed");
};
return match method {
"GetCacheEntryDownloadURL" if served => lookup(inner, runner, &host, &body),
"GetCacheEntryDownloadURL" => missed(),
"CreateCacheEntry" if served => create(inner, runner, &host, &body),
"CreateCacheEntry" => created(&host, DROPPED),
"FinalizeCacheEntryUpload" => finalize(inner, runner, &body),
_ => twirp_error(StatusCode::NOT_FOUND, "bad_route"),
};
}
if path.starts_with(TWIRP) {
return pass(inner, request).await;
}
if let Some(id) = path.strip_prefix(UPLOADS) {
let id = id.parse().unwrap_or(0);
return upload(inner, runner, id, request).await;
}
if let Some(id) = path.strip_prefix(DOWNLOADS)
&& let Ok(id) = id.parse()
{
return download(inner, runner, id, &request).await;
}
status(StatusCode::NOT_FOUND)
}
#[derive(Debug, Deserialize)]
struct LookupRequest {
key: String,
#[serde(default, alias = "restoreKeys")]
restore_keys: Vec<String>,
version: String,
}
#[derive(Debug, Deserialize)]
struct CreateRequest {
key: String,
version: String,
}
#[derive(Debug, Deserialize)]
struct FinalizeRequest {
key: String,
version: String,
#[serde(default, alias = "sizeBytes")]
size_bytes: serde_json::Value,
}
fn lookup(inner: &Inner, runner: &str, host: &str, body: &[u8]) -> Response<Body> {
let Ok(request) = serde_json::from_slice::<LookupRequest>(body) else {
return twirp_error(StatusCode::BAD_REQUEST, "malformed");
};
let mut state = lock(&inner.state);
let Some(open) = state.runners.get(runner) else {
return status(StatusCode::SERVICE_UNAVAILABLE);
};
let scopes = read_scopes(open, state.default_branches.get(&open.repository));
let keys: Vec<&str> = std::iter::once(request.key.as_str())
.chain(request.restore_keys.iter().map(String::as_str))
.filter(|key| valid_key(key))
.collect();
let found = if valid_segment(&request.version) {
scopes.iter().find_map(|reference| {
let scope = scope_dir(&open.repository, reference, &request.version);
let paths = state.actions.get(&scope)?;
keys.iter()
.find_map(|key| newest(paths.iter(), &scope, key))
})
} else {
None
};
let Some(open) = state.runners.get_mut(runner) else {
return status(StatusCode::SERVICE_UNAVAILABLE);
};
let found = found.filter(|found| {
open.found.contains_key(&found.created)
|| open.found.len() < MAX_FOUND
&& open
.found
.insert(found.created, found.path.clone())
.is_none()
});
let Some(Found { path, key, created }) = found else {
open.counts.misses += 1;
return missed();
};
open.counts.hits += 1;
drop(state);
inner.touch(&path);
json(&serde_json::json!({
"ok": true,
"signed_download_url": format!("http://{host}/{DOWNLOADS}{created}"),
"matched_key": key,
}))
}
fn missed() -> Response<Body> {
json(&serde_json::json!({
"ok": false,
"signed_download_url": "",
"matched_key": "",
}))
}
fn read_scopes(open: &Runner, default_branch: Option<&String>) -> Vec<String> {
let mut scopes = Vec::new();
if let Some(allowed) = open.writes.allowed() {
scopes.extend(allowed.own.iter().chain(&allowed.base).cloned());
}
if let Some(default) = open.writes.default_branch().or(default_branch) {
scopes.push(default.clone());
}
scopes.dedup();
scopes
}
struct Found {
path: PathBuf,
key: String,
created: u64,
}
fn newest<'entries>(
entries: impl Iterator<Item = &'entries PathBuf>,
scope: &Path,
key: &str,
) -> Option<Found> {
let mut exact: Option<(u64, &PathBuf, String)> = None;
let mut prefixed: Option<(u64, &PathBuf, String)> = None;
for path in entries {
let Some((found, created)) = parse_entry(path, scope) else {
continue;
};
let best = if found == key {
&mut exact
} else if found.starts_with(key) {
&mut prefixed
} else {
continue;
};
if best.as_ref().is_none_or(|(newest, _, _)| created > *newest) {
*best = Some((created, path, found));
}
}
exact.or(prefixed).map(|(created, path, key)| Found {
path: path.clone(),
key,
created,
})
}
fn create(inner: &Inner, runner: &str, host: &str, body: &[u8]) -> Response<Body> {
let Ok(request) = serde_json::from_slice::<CreateRequest>(body) else {
return twirp_error(StatusCode::BAD_REQUEST, "malformed");
};
if !valid_key(&request.key) || !valid_segment(&request.version) {
return twirp_error(StatusCode::BAD_REQUEST, "invalid_argument");
}
let mut state = lock(&inner.state);
let Some(open) = state.runners.get_mut(runner) else {
return status(StatusCode::SERVICE_UNAVAILABLE);
};
let writable = open.saves.len() < MAX_SAVES
&& match open.writes.current() {
Writes::Undecided => true,
Writes::Allowed => open.writes.own().is_some(),
Writes::Refused => false,
};
if !writable {
open.counts.refused += 1;
return created(host, DROPPED);
}
if let Some(to) = destination(open, &request.key, &request.version) {
let saving = state.runners.values().any(|other| {
other.saves.values().any(|save| {
!save.is_dropped()
&& destination(other, save.key(), save.version())
.is_some_and(|theirs| theirs.dir == to.dir)
})
});
if saving || stored_already(&state, &to) {
return created(host, DROPPED);
}
}
let max_size = state.max_size;
state.next_upload += 1;
let id = state.next_upload;
if let Some(open) = state.runners.get_mut(runner) {
open.saves
.insert(id, Save::new(request.key, request.version, max_size));
}
created(host, id)
}
const DROPPED: u64 = 0;
fn created(host: &str, id: u64) -> Response<Body> {
json(&serde_json::json!({
"ok": true,
"signed_upload_url": format!("http://{host}/{UPLOADS}{id}"),
"message": "",
}))
}
fn finalize(inner: &Inner, runner: &str, body: &[u8]) -> Response<Body> {
let Ok(request) = serde_json::from_slice::<FinalizeRequest>(body) else {
return twirp_error(StatusCode::BAD_REQUEST, "malformed");
};
let size = match &request.size_bytes {
serde_json::Value::Number(number) => number.as_u64(),
serde_json::Value::String(text) => text.parse().ok(),
_ => None,
};
let stored = |entry_id: u64| {
json(&serde_json::json!({ "ok": true, "entry_id": entry_id.to_string(), "message": "" }))
};
let not_stored = || json(&serde_json::json!({ "ok": false, "entry_id": "0", "message": "" }));
let mut gone = Vec::new();
let mut state = lock(&inner.state);
let Some(open) = state.runners.get_mut(runner) else {
return status(StatusCode::SERVICE_UNAVAILABLE);
};
let found = open
.saves
.iter()
.find(|(_, save)| save.answers_finalize(&request.key, &request.version))
.map(|(id, _)| *id);
let Some((id, mut save)) = found.and_then(|id| Some((id, open.saves.remove(&id)?))) else {
return stored(0);
};
let answer = match save.finalize(size, &mut gone) {
Finished::Archived if open.writes.is_undecided() => {
open.saves.insert(id, save);
stored(0)
}
Finished::Archived => {
stored(publish(inner, &mut state, runner, save, &mut gone).unwrap_or(0))
}
Finished::WrongSize => not_stored(),
Finished::Nothing => stored(0),
};
drop(state);
inner.remove_uploads(Some(runner), &gone);
answer
}
fn destination(open: &Runner, key: &str, version: &str) -> Option<Destination> {
let own = open.writes.own()?;
Some(Destination {
scope: scope_dir(&open.repository, own, version),
dir: entry_dir(&open.repository, own, version, key),
})
}
struct Destination {
scope: PathBuf,
dir: PathBuf,
}
fn stored_already(state: &State, to: &Destination) -> bool {
state.actions.get(&to.scope).is_some_and(|paths| {
paths
.iter()
.any(|path| path.parent() == Some(to.dir.as_path()))
})
}
fn publish(
inner: &Inner,
state: &mut State,
runner: &str,
save: Save,
gone: &mut Vec<Upload>,
) -> Option<u64> {
let to = state
.runners
.get(runner)
.and_then(|open| destination(open, save.key(), save.version()));
let mut archive = save.into_archive()?;
let Some(to) = to.filter(|to| !stored_already(state, to)) else {
gone.push(archive);
return None;
};
let created = crate::time::now_ms().max(state.last_created + 1);
state.last_created = created;
archive.entry = to.dir.join(format!("{END}{created}"));
inner
.store_locked(state, runner, archive)
.then_some(created)
}
pub(super) fn decided(inner: &Inner, state: &mut State, runner: &str) -> Vec<Upload> {
let mut gone = Vec::new();
let Some(open) = state.runners.get_mut(runner) else {
return gone;
};
if open.writes.own().is_some() {
let mut finalized: Vec<u64> = open
.saves
.iter()
.filter(|(_, save)| save.is_finalized())
.map(|(id, _)| *id)
.collect();
finalized.sort_unstable();
for id in finalized {
if let Some(save) = state
.runners
.get_mut(runner)
.and_then(|open| open.saves.remove(&id))
{
publish(inner, state, runner, save, &mut gone);
}
}
return gone;
}
let mut refused = 0;
open.saves.retain(|_, save| {
if save.is_dropped() {
return true;
}
refused += 1;
let answers = !save.is_finalized();
save.drop_uploads(&mut gone);
answers
});
open.counts.refused += refused;
gone
}
pub(super) fn finalized(saves: &HashMap<u64, Save>) -> u64 {
saves.values().filter(|save| save.is_finalized()).count() as u64
}
pub(super) fn unfinished(saves: &mut HashMap<u64, Save>) -> HashMap<u64, Save> {
let ids: Vec<u64> = saves
.iter()
.filter(|(_, save)| !save.is_finalized())
.map(|(id, _)| *id)
.collect();
ids.into_iter()
.filter_map(|id| Some((id, saves.remove(&id)?)))
.collect()
}
async fn upload(
inner: &Arc<Inner>,
runner: &str,
id: u64,
request: Request<Incoming>,
) -> Response<Body> {
if request.method() != Method::PUT {
return status(StatusCode::METHOD_NOT_ALLOWED);
}
let query: Vec<(String, String)> = request
.uri()
.query()
.and_then(|query| serde_urlencoded::from_str(query).ok())
.unwrap_or_default();
let field = |name: &str| {
query
.iter()
.find(|(key, _)| key == name)
.map(|(_, value)| value.clone())
};
let (comp, block) = (field("comp"), field("blockid"));
let max_size = lock(&inner.state)
.runners
.get(runner)
.and_then(|open| open.saves.get(&id))
.filter(|save| save.is_open())
.map(Save::max_size);
let body = request.into_body();
let Some(max_size) = max_size else {
drain(body).await;
return blob_created();
};
match (comp.as_deref(), block) {
(Some("blocklist"), _) => join(inner, runner, id, max_size, body).await,
(Some("block"), Some(block)) if !block.is_empty() && block.len() <= 128 => {
let Some(received) = take(inner, runner, id, max_size, body).await else {
return status(StatusCode::INTERNAL_SERVER_ERROR);
};
keep(inner, runner, id, received, Part::Block(block));
blob_created()
}
(Some("block"), _) => status(StatusCode::BAD_REQUEST),
_ => {
let Some(received) = take(inner, runner, id, max_size, body).await else {
return status(StatusCode::INTERNAL_SERVER_ERROR);
};
keep(inner, runner, id, received, Part::Archive);
blob_created()
}
}
}
async fn take(
inner: &Inner,
runner: &str,
id: u64,
max_size: u64,
body: Incoming,
) -> Option<Option<Upload>> {
let mut receiving = Receiving::new(inner, runner, max_size, Purpose::Save(id));
match receive(&mut receiving, body).await {
Ok(true) => Some(Some(receiving.into_upload(PathBuf::new()))),
Ok(false) => Some(None),
Err(error) => {
debug!("{runner}: an actions cache upload failed: {error}");
None
}
}
}
fn keep(inner: &Inner, runner: &str, id: u64, received: Option<Upload>, part: Part) {
let mut gone = Vec::new();
{
let mut state = lock(&inner.state);
let Some(open) = state.runners.get_mut(runner) else {
drop(state);
inner.remove_uploads(None, &received.into_iter().collect::<Vec<_>>());
return;
};
let Some(save) = open.saves.get_mut(&id).filter(|save| save.is_open()) else {
gone.extend(received);
drop(state);
inner.remove_uploads(Some(runner), &gone);
return;
};
save.keep(received, part, &mut gone);
if save.is_dropped() {
open.counts.refused += 1;
}
}
inner.remove_uploads(Some(runner), &gone);
}
async fn join(
inner: &Arc<Inner>,
runner: &str,
id: u64,
max_size: u64,
body: Incoming,
) -> Response<Body> {
let Some(list) = read(body, MAX_BLOCK_LIST).await else {
return status(StatusCode::BAD_REQUEST);
};
let Some(order) = block_ids(&String::from_utf8_lossy(&list)) else {
return status(StatusCode::BAD_REQUEST);
};
let blocks = {
let mut state = lock(&inner.state);
let Some(save) = state
.runners
.get_mut(runner)
.and_then(|open| open.saves.get_mut(&id))
else {
return blob_created();
};
save.take_blocks()
};
let mut blocks = blocks;
let mut parts = Vec::with_capacity(order.len());
for block in &order {
let Some(upload) = blocks.remove(block) else {
parts.extend(blocks.into_values());
inner.remove_uploads(Some(runner), &parts);
keep(inner, runner, id, None, Part::Archive);
return status(StatusCode::BAD_REQUEST);
};
parts.push(upload);
}
inner.remove_uploads(Some(runner), &blocks.into_values().collect::<Vec<_>>());
if parts.iter().map(|part| part.bytes).sum::<u64>() > max_size / 4 {
inner.remove_uploads(Some(runner), &parts);
keep(inner, runner, id, None, Part::Archive);
return blob_created();
}
let file = {
let mut state = lock(&inner.state);
state.next_upload += 1;
inner.uploads.join(state.next_upload.to_string())
};
let (inner, runner) = (inner.clone(), runner.to_owned());
let joining = Joining {
id,
max_size,
parts,
file,
};
match tokio::task::spawn_blocking(move || joining.run(&inner, &runner)).await {
Ok(true) => blob_created(),
_ => status(StatusCode::INTERNAL_SERVER_ERROR),
}
}
struct Joining {
id: u64,
max_size: u64,
parts: Vec<Upload>,
file: PathBuf,
}
impl Joining {
fn run(self, inner: &Inner, runner: &str) -> bool {
#[cfg(test)]
inner.join_gate.pass();
let Joining {
id,
max_size,
parts,
file,
} = self;
let joined = (|| -> io::Result<u64> {
let mut out = std::fs::File::create(&file)?;
let mut bytes = 0;
for part in &parts {
bytes += io::copy(&mut std::fs::File::open(&part.file)?, &mut out)?;
}
out.sync_data()?;
Ok(bytes)
})();
let room: u64 = parts.iter().map(|part| part.room).sum();
for part in &parts {
let _ = std::fs::remove_file(&part.file);
}
let needed = joined.as_ref().map_or(0, |bytes| charge(*bytes));
let kept = match joined {
Ok(bytes) if room >= needed => {
inner.release(Some(runner), room - needed);
Some(bytes)
}
Ok(bytes) if inner.reserve(runner, needed - room, max_size) => Some(bytes),
_ => None,
};
let Some(bytes) = kept else {
let _ = std::fs::remove_file(&file);
inner.release(Some(runner), room);
keep(inner, runner, id, None, Part::Archive);
return joined.is_ok();
};
let archive = Upload {
file,
entry: PathBuf::new(),
bytes,
room: needed,
};
keep(inner, runner, id, Some(archive), Part::Archive);
true
}
}
pub(super) fn takes_uploads(saves: &HashMap<u64, Save>, id: u64) -> bool {
saves.get(&id).is_some_and(Save::is_open)
}
fn block_ids(list: &str) -> Option<Vec<String>> {
let mut ids = Vec::new();
let mut rest = list;
while let Some(start) = rest.find('<') {
rest = &rest[start + 1..];
let end = rest.find('>')?;
let tag = &rest[..end];
rest = &rest[end + 1..];
if matches!(tag, "Latest" | "Committed" | "Uncommitted") {
let close = rest.find("</")?;
ids.push(rest[..close].trim().to_owned());
rest = &rest[close..];
}
}
Some(ids)
}
async fn download(
inner: &Inner,
runner: &str,
id: u64,
request: &Request<Incoming>,
) -> Response<Body> {
if request.method() != Method::GET && request.method() != Method::HEAD {
return status(StatusCode::METHOD_NOT_ALLOWED);
}
let Some(entry) = lock(&inner.state)
.runners
.get(runner)
.and_then(|open| open.found.get(&id).cloned())
else {
return status(StatusCode::NOT_FOUND);
};
let Ok(mut file) = tokio::fs::File::open(inner.root.join(&entry)).await else {
return status(StatusCode::NOT_FOUND);
};
let Ok(size) = file.metadata().await.map(|metadata| metadata.len()) else {
return status(StatusCode::NOT_FOUND);
};
let range = request
.headers()
.get(MS_RANGE)
.or_else(|| request.headers().get(hyper::header::RANGE))
.and_then(|value| value.to_str().ok())
.and_then(parse_range);
let (start, length, partial) = match range {
None => (0, size, false),
Some((start, _)) if start >= size => {
let mut response = status(StatusCode::RANGE_NOT_SATISFIABLE);
if let Ok(value) = HeaderValue::from_str(&format!("bytes */{size}")) {
response.headers_mut().insert(CONTENT_RANGE, value);
}
return response;
}
Some((start, end)) => {
let end = end.map_or(size - 1, |end| end.min(size - 1));
(start, end - start + 1, true)
}
};
let head = request.method() == Method::HEAD;
if !head && file.seek(io::SeekFrom::Start(start)).await.is_err() {
return status(StatusCode::INTERNAL_SERVER_ERROR);
}
if !head && let Some(open) = lock(&inner.state).runners.get_mut(runner) {
open.counts.read_bytes += length;
}
let body = if head {
Empty::new().map_err(|never| match never {}).boxed_unsync()
} else {
let chunks = futures_util::stream::unfold((file, length), |(mut file, left)| async move {
if left == 0 {
return None;
}
let mut chunk = vec![0; (64 << 10).min(usize::try_from(left).unwrap_or(usize::MAX))];
match file.read(&mut chunk).await {
Ok(0) => None,
Ok(read) => {
chunk.truncate(read);
Some((
Ok(Frame::data(Bytes::from(chunk))),
(file, left - read as u64),
))
}
Err(error) => Some((Err(error), (file, 0))),
}
});
BodyExt::boxed_unsync(StreamBody::new(chunks))
};
let mut response = Response::new(body);
if partial {
*response.status_mut() = StatusCode::PARTIAL_CONTENT;
if let Ok(value) =
HeaderValue::from_str(&format!("bytes {start}-{}/{size}", start + length - 1))
{
response.headers_mut().insert(CONTENT_RANGE, value);
}
}
let headers = response.headers_mut();
headers.insert(CONTENT_LENGTH, length.into());
headers.insert(
hyper::header::ACCEPT_RANGES,
HeaderValue::from_static("bytes"),
);
headers.insert(REQUEST_ID, request_id());
response
}
fn parse_range(value: &str) -> Option<(u64, Option<u64>)> {
let (start, end) = value.trim().strip_prefix("bytes=")?.split_once('-')?;
let start = start.trim().parse().ok()?;
let end = match end.trim() {
"" => None,
end => Some(end.parse().ok()?),
};
end.is_none_or(|end| end >= start).then_some((start, end))
}
async fn pass(inner: &Inner, request: Request<Incoming>) -> Response<Body> {
let (parts, body) = request.into_parts();
let Some(body) = read(body, MAX_PASSED).await else {
return status(StatusCode::PAYLOAD_TOO_LARGE);
};
let Some(client) = inner.results.client() else {
return status(StatusCode::BAD_GATEWAY);
};
let url = format!(
"{}{}",
lock(&inner.results.origin),
parts.uri.path_and_query().map_or("/", |path| path.as_str())
);
let sent = client
.request(parts.method, url)
.headers(passed_headers(&parts.headers))
.body(body)
.send()
.await;
let mut answer = match sent {
Ok(answer) => answer,
Err(error) => {
debug!(
"cannot pass {} to GitHub's results service: {}",
parts.uri.path(),
error.without_url()
);
return status(StatusCode::BAD_GATEWAY);
}
};
let code = answer.status();
let headers = passed_headers(answer.headers());
let mut bytes = Vec::new();
loop {
match answer.chunk().await {
Ok(Some(chunk)) if bytes.len() + chunk.len() <= MAX_PASSED => {
bytes.extend_from_slice(&chunk);
}
Ok(None) => break,
_ => return status(StatusCode::BAD_GATEWAY),
}
}
let mut response = Response::new(
Full::new(Bytes::from(bytes))
.map_err(|never| match never {})
.boxed_unsync(),
);
*response.status_mut() = code;
*response.headers_mut() = headers;
response
}
pub(super) fn passed_headers(headers: &HeaderMap) -> HeaderMap {
const OWN: [&str; 10] = [
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"host",
"content-length",
];
headers
.iter()
.filter(|(name, _)| !OWN.contains(&name.as_str()))
.map(|(name, value)| (name.clone(), value.clone()))
.collect()
}
fn host(headers: &HeaderMap) -> String {
headers
.get(HOST)
.and_then(|value| value.to_str().ok())
.filter(|host| {
!host.is_empty()
&& host.len() <= 64
&& host
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b".:-[]".contains(&byte))
})
.unwrap_or(DEFAULT_HOST)
.to_owned()
}
fn valid_key(key: &str) -> bool {
!key.is_empty() && key.len() <= MAX_KEY && !key.contains(',')
}
fn scope_dir(repository: &str, scope: &str, version: &str) -> PathBuf {
let mut path = PathBuf::from(repository);
path.push(TOOL);
push_hex(&mut path, scope);
path.push(END);
path.push(version);
path
}
fn entry_dir(repository: &str, scope: &str, version: &str, key: &str) -> PathBuf {
let mut path = scope_dir(repository, scope, version);
push_hex(&mut path, key);
path
}
fn push_hex(path: &mut PathBuf, text: &str) {
const DIGITS: &[u8; 16] = b"0123456789abcdef";
let hex: String = text
.bytes()
.flat_map(|byte| {
[
DIGITS[usize::from(byte >> 4)],
DIGITS[usize::from(byte & 15)],
]
})
.map(char::from)
.collect();
for segment in hex.as_bytes().chunks(SEGMENT) {
path.push(String::from_utf8_lossy(segment).as_ref());
}
}
fn parse_entry(path: &Path, scope: &Path) -> Option<(String, u64)> {
let rest = path.strip_prefix(scope).ok()?;
let segments: Vec<&str> = rest
.iter()
.map(|segment| segment.to_str())
.collect::<Option<_>>()?;
let (last, hex) = segments.split_last()?;
let created = last.strip_prefix(END)?.parse().ok()?;
let hex = hex.concat();
let bytes = (0..hex.len())
.step_by(2)
.map(|at| u8::from_str_radix(hex.get(at..at + 2)?, 16).ok())
.collect::<Option<Vec<u8>>>()?;
Some((String::from_utf8(bytes).ok()?, created))
}
pub(super) fn scope_of(path: &Path) -> Option<PathBuf> {
let segments: Vec<&std::ffi::OsStr> = path.iter().collect();
if *segments.get(2)? != TOOL {
return None;
}
let end = 3 + segments
.get(3..)?
.iter()
.position(|segment| *segment == END)?;
(segments.len() > end + 3).then(|| segments[..end + 2].iter().collect())
}
pub(super) fn created_of(path: &Path) -> Option<u64> {
scope_of(path)?;
path.file_name()?.to_str()?.strip_prefix(END)?.parse().ok()
}
pub(super) async fn read(body: Incoming, limit: usize) -> Option<Bytes> {
Limited::new(body, limit)
.collect()
.await
.ok()
.map(http_body_util::Collected::to_bytes)
}
pub(super) fn json(value: &serde_json::Value) -> Response<Body> {
let mut response = Response::new(
Full::new(Bytes::from(value.to_string()))
.map_err(|never| match never {})
.boxed_unsync(),
);
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
response
}
fn twirp_error(code: StatusCode, kind: &str) -> Response<Body> {
let mut response = json(&serde_json::json!({ "code": kind, "msg": kind }));
*response.status_mut() = code;
response
}
fn blob_created() -> Response<Body> {
let mut response = status(StatusCode::CREATED);
response.headers_mut().insert(REQUEST_ID, request_id());
response
}
fn request_id() -> HeaderValue {
HeaderValue::from_static("00000000-0000-0000-0000-000000000000")
}
#[cfg(test)]
mod tests;
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct Answer {
pub(crate) status: u16,
pub(crate) headers: Vec<(String, String)>,
pub(crate) body: Vec<u8>,
}
#[cfg(test)]
impl Answer {
pub(crate) fn header(&self, name: &str) -> Option<&str> {
self.headers
.iter()
.find(|(key, _)| key == name)
.map(|(_, value)| value.as_str())
}
pub(crate) fn json(&self) -> serde_json::Value {
serde_json::from_slice(&self.body).unwrap_or_default()
}
}
#[cfg(test)]
#[allow(
clippy::unwrap_used,
reason = "a test helper: a socket that does not answer fails the test"
)]
pub(crate) async fn send(
dir: &Path,
method: &str,
path: &str,
headers: &[(&str, &str)],
body: &[u8],
) -> Answer {
use tokio::io::AsyncWriteExt;
let mut stream = tokio::net::UnixStream::connect(dir.join(super::SOCKET))
.await
.unwrap();
let mut head = format!(
"{method} {path} HTTP/1.1\r\nHost: 10.0.0.5:41772\r\nContent-Length: {}\r\nConnection: close\r\n",
body.len()
);
for (name, value) in headers {
head.extend([name, ": ", value, "\r\n"]);
}
head.push_str("\r\n");
stream.write_all(head.as_bytes()).await.unwrap();
stream.write_all(body).await.unwrap();
let mut response = Vec::new();
stream.read_to_end(&mut response).await.unwrap();
let split = response
.windows(4)
.position(|window| window == b"\r\n\r\n")
.unwrap();
let head = String::from_utf8_lossy(&response[..split]).into_owned();
let mut lines = head.lines();
let status = lines.next().unwrap()[9..12].parse().unwrap();
let headers = lines
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.trim().to_ascii_lowercase(), value.trim().to_owned()))
.collect();
Answer {
status,
headers,
body: response[split + 4..].to_vec(),
}
}
#[cfg(test)]
pub(crate) async fn call(
dir: &Path,
method: &str,
request: &serde_json::Value,
) -> serde_json::Value {
send(
dir,
"POST",
&format!("/{CACHE_SERVICE}{method}"),
&[
("Authorization", "Bearer runtime-token"),
("Content-Type", "application/json"),
],
request.to_string().as_bytes(),
)
.await
.json()
}
#[cfg(test)]
pub(crate) fn path_of(url: &serde_json::Value) -> String {
let url = url.as_str().unwrap_or_default();
url.strip_prefix("http://10.0.0.5:41772")
.unwrap_or(url)
.to_owned()
}
#[cfg(test)]
pub(crate) async fn save(dir: &Path, key: &str, archive: &[u8]) -> bool {
let reserved = call(
dir,
"CreateCacheEntry",
&serde_json::json!({ "key": key, "version": VERSION }),
)
.await;
if reserved["ok"] != true {
return false;
}
let sent = send(
dir,
"PUT",
&path_of(&reserved["signed_upload_url"]),
&[("x-ms-blob-type", "BlockBlob")],
archive,
)
.await;
let finalized = call(
dir,
"FinalizeCacheEntryUpload",
&serde_json::json!({ "key": key, "version": VERSION, "size_bytes": archive.len().to_string() }),
)
.await;
sent.status == 201 && finalized["ok"] == true
}
#[cfg(test)]
pub(crate) async fn restore(
dir: &Path,
key: &str,
restore_keys: &[&str],
) -> Option<(String, Vec<u8>)> {
let found = call(
dir,
"GetCacheEntryDownloadURL",
&serde_json::json!({ "key": key, "restore_keys": restore_keys, "version": VERSION }),
)
.await;
if found["ok"] != true {
return None;
}
let archive = send(
dir,
"GET",
&path_of(&found["signed_download_url"]),
&[],
b"",
)
.await;
assert_eq!(archive.status, 200, "{archive:?}");
Some((found["matched_key"].as_str()?.to_owned(), archive.body))
}
#[cfg(test)]
pub(crate) const VERSION: &str = "b4f2e8a1c3d5e7f9a1b3c5d7e9f1a3b5c7d9e1f3a5b7c9d1e3f5a7b9c1d3e5f7";