use std::io;
use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError};
use std::time::{Duration, Instant};
use bytes::Buf;
use crate::operation::download::body::DiskWriteCursor;
use crate::operation::download::sink::{FileSinkFactory, SinkFactory, SinkWrite};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Match {
NthWrite(usize),
WriteAtOrAfter(u64),
Finalize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Action {
Hold,
Fail(io::ErrorKind),
Delay(Duration),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Op {
Write { pos: u64, len: usize },
Finalize { len: u64 },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Event {
Started(Op),
Finished(Op, Result<(), io::ErrorKind>),
Mark(&'static str),
}
#[derive(Debug, Default)]
pub(crate) struct WriteScript {
state: Mutex<ScriptState>,
changed: Condvar,
}
#[derive(Debug, Default)]
struct ScriptState {
rules: Vec<(Match, Action)>,
writes: usize,
log: Vec<Event>,
holds: Vec<HoldSlot>,
}
#[derive(Debug)]
struct HoldSlot {
op: Op,
claimed: bool,
release: Option<Result<(), io::ErrorKind>>,
}
impl ScriptState {
fn take_rule(&mut self, op: Op, write_number: usize) -> Option<Action> {
let index = self.rules.iter().position(|(m, _)| match (*m, op) {
(Match::NthWrite(n), Op::Write { .. }) => n == write_number,
(Match::WriteAtOrAfter(offset), Op::Write { pos, .. }) => pos >= offset,
(Match::Finalize, Op::Finalize { .. }) => true,
_ => false,
})?;
Some(self.rules.remove(index).1)
}
}
fn scripted_error(kind: io::ErrorKind) -> io::Error {
io::Error::new(kind, "scripted destination failure")
}
impl WriteScript {
pub(crate) fn on(self: &Arc<Self>, m: Match, a: Action) -> Arc<Self> {
self.lock().rules.push((m, a));
Arc::clone(self)
}
pub(crate) fn wait_held(self: &Arc<Self>, timeout: Duration) -> Held {
let deadline = Instant::now() + timeout;
let mut state = self.lock();
loop {
if let Some(id) = state.holds.iter().position(|hold| !hold.claimed) {
state.holds[id].claimed = true;
return Held {
script: Arc::clone(self),
id,
op: state.holds[id].op,
released: false,
};
}
let now = Instant::now();
assert!(
now < deadline,
"no destination operation was held within {timeout:?}"
);
state = self
.changed
.wait_timeout(state, deadline - now)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
}
pub(crate) fn mark(&self, label: &'static str) {
self.lock().log.push(Event::Mark(label));
}
pub(crate) fn events(&self) -> Vec<Event> {
self.lock().log.clone()
}
fn run(&self, op: Op, io: impl FnOnce() -> io::Result<()>) -> io::Result<()> {
let result = self.before(op).and_then(|()| io());
self.lock().log.push(Event::Finished(
op,
result.as_ref().map(|_| ()).map_err(io::Error::kind),
));
result
}
fn before(&self, op: Op) -> io::Result<()> {
let mut state = self.lock();
state.log.push(Event::Started(op));
if let Op::Write { .. } = op {
state.writes += 1;
}
let write_number = state.writes;
match state.take_rule(op, write_number) {
None => Ok(()),
Some(Action::Fail(kind)) => Err(scripted_error(kind)),
Some(Action::Delay(delay)) => {
drop(state);
std::thread::sleep(delay);
Ok(())
}
Some(Action::Hold) => {
let id = state.holds.len();
state.holds.push(HoldSlot {
op,
claimed: false,
release: None,
});
self.changed.notify_all();
loop {
if let Some(release) = state.holds[id].release {
return release.map_err(scripted_error);
}
state = self
.changed
.wait(state)
.unwrap_or_else(PoisonError::into_inner);
}
}
}
}
fn lock(&self) -> MutexGuard<'_, ScriptState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
#[derive(Debug)]
pub(crate) struct Held {
script: Arc<WriteScript>,
id: usize,
op: Op,
released: bool,
}
impl Held {
pub(crate) fn op(&self) -> Op {
self.op
}
pub(crate) fn release(mut self) {
self.finish(Ok(()));
}
pub(crate) fn release_err(mut self, kind: io::ErrorKind) {
self.finish(Err(kind));
}
fn finish(&mut self, release: Result<(), io::ErrorKind>) {
self.released = true;
self.script.lock().holds[self.id].release = Some(release);
self.script.changed.notify_all();
}
}
impl Drop for Held {
fn drop(&mut self) {
if !self.released {
self.finish(Ok(()));
}
}
}
#[derive(Debug)]
pub(crate) struct ScriptedSinkFactory(pub(crate) Arc<WriteScript>);
impl SinkFactory for ScriptedSinkFactory {
fn create(&self, file: std::fs::File, owns_file: bool) -> io::Result<Box<dyn SinkWrite>> {
Ok(Box::new(ScriptedSink {
inner: FileSinkFactory.create(file, owns_file)?,
script: Arc::clone(&self.0),
}))
}
}
#[derive(Debug)]
struct ScriptedSink {
inner: Box<dyn SinkWrite>,
script: Arc<WriteScript>,
}
impl SinkWrite for ScriptedSink {
fn write_all_at(&self, buf: &mut DiskWriteCursor<'_>, pos: u64) -> io::Result<()> {
let op = Op::Write {
pos,
len: buf.remaining(),
};
self.script.run(op, || self.inner.write_all_at(buf, pos))
}
fn prepare(&self, expected_download_len: u64) -> io::Result<()> {
self.inner.prepare(expected_download_len)
}
fn finalize(&self, expected_download_len: u64) -> io::Result<()> {
let op = Op::Finalize {
len: expected_download_len,
};
self.script
.run(op, || self.inner.finalize(expected_download_len))
}
}
#[cfg(test)]
mod tests {
use crate::types::ByteSize;
use std::io;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use super::{Action, Event, Match, Op, ScriptedSinkFactory, WriteScript};
use crate::memory::SegmentedBytes;
use crate::operation::download::body::{new_recv_body_with_disk_mode, BodyWriter, ChunkOutput};
use crate::operation::download::recv_buffer::DrainMode;
use crate::operation::download::sink::SinkFactory;
use crate::operation::download::test_util::fixtures::{
managed_test_handle, object_bytes, object_client,
};
use crate::operation::download::{Download, DownloadInput};
const HOLD_TIMEOUT: Duration = Duration::from_secs(5);
fn scripted_writer(
script: &Arc<WriteScript>,
file: std::fs::File,
expected_len: u64,
) -> BodyWriter {
let sink = ScriptedSinkFactory(Arc::clone(script))
.create(file, false)
.unwrap();
let (writer, _consumer) = new_recv_body_with_disk_mode(sink);
writer.prepare(0, expected_len).unwrap();
writer
}
fn fill(writer: &BodyWriter, offset: u64, data: &'static [u8]) {
let slot = writer.claim();
let seq = slot.seq();
slot.fill(ChunkOutput {
seq,
offset,
data: SegmentedBytes::from(Bytes::from_static(data)),
metadata: Default::default(),
});
}
#[test]
fn held_write_blocks_until_released() {
let script = Arc::new(WriteScript::default()).on(Match::NthWrite(1), Action::Hold);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let writer = scripted_writer(&script, std::fs::File::create(&path).unwrap(), 4);
fill(&writer, 0, b"abcd");
std::thread::scope(|scope| {
let drain = scope.spawn(|| writer.drain(DrainMode::Eager));
let held = script.wait_held(HOLD_TIMEOUT);
assert_eq!(held.op(), Op::Write { pos: 0, len: 4 });
assert!(
std::fs::read(&path).unwrap().is_empty(),
"a held write must not reach the file"
);
script.mark("held");
held.release();
drain.join().unwrap().unwrap();
});
assert_eq!(std::fs::read(&path).unwrap(), b"abcd");
let write = Op::Write { pos: 0, len: 4 };
assert_eq!(
script.events(),
[
Event::Started(write),
Event::Mark("held"),
Event::Finished(write, Ok(())),
]
);
}
#[test]
fn held_write_released_with_error_fails() {
let script = Arc::new(WriteScript::default()).on(Match::NthWrite(1), Action::Hold);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let writer = scripted_writer(&script, std::fs::File::create(&path).unwrap(), 4);
fill(&writer, 0, b"abcd");
let error = std::thread::scope(|scope| {
let drain = scope.spawn(|| writer.drain(DrainMode::Eager));
script
.wait_held(HOLD_TIMEOUT)
.release_err(io::ErrorKind::StorageFull);
drain.join().unwrap().expect_err("released with an error")
});
assert_eq!(error.source.kind(), io::ErrorKind::StorageFull);
assert!(std::fs::read(&path).unwrap().is_empty());
}
#[test]
fn fail_rule_matches_first_write_at_or_after_offset() {
let script = Arc::new(WriteScript::default()).on(
Match::WriteAtOrAfter(2),
Action::Fail(io::ErrorKind::StorageFull),
);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let writer = scripted_writer(&script, std::fs::File::create(&path).unwrap(), 6);
fill(&writer, 0, b"ab");
writer.drain(DrainMode::Eager).unwrap();
fill(&writer, 2, b"cd");
let error = writer
.drain(DrainMode::Eager)
.expect_err("the write at offset 2 is scripted to fail");
assert_eq!(error.source.kind(), io::ErrorKind::StorageFull);
fill(&writer, 4, b"ef");
writer.drain(DrainMode::Eager).unwrap();
let contents = std::fs::read(&path).unwrap();
assert_eq!(&contents[..2], b"ab");
assert_eq!(&contents[2..4], [0, 0], "the failed write must not land");
assert_eq!(&contents[4..], b"ef");
let failed = Op::Write { pos: 2, len: 2 };
assert!(script
.events()
.contains(&Event::Finished(failed, Err(io::ErrorKind::StorageFull))));
}
#[test]
fn finalize_rule_fails_the_resize() {
let script = Arc::new(WriteScript::default()).on(
Match::Finalize,
Action::Fail(io::ErrorKind::PermissionDenied),
);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::write(&path, b"stale contents").unwrap();
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.unwrap();
let writer = scripted_writer(&script, file, 3);
fill(&writer, 0, b"new");
let error = writer
.finalize(3)
.expect_err("finalize is scripted to fail");
assert_eq!(error.source.kind(), io::ErrorKind::PermissionDenied);
assert_eq!(std::fs::read(&path).unwrap(), b"newle contents");
assert_eq!(
script.events().last(),
Some(&Event::Finished(
Op::Finalize { len: 3 },
Err(io::ErrorKind::PermissionDenied)
))
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn path_download_writes_through_its_sink_factory() {
const MIB: usize = 1024 * 1024;
let object = object_bytes(5 * MIB + 1024);
let script = Arc::new(WriteScript::default());
let config = crate::Config::builder()
.sdk_client(object_client(Arc::clone(&object)))
.part_size(crate::config::PartSize::Target(ByteSize::new(
5 * MIB as u64,
)))
.build()
.expect("valid config");
let handle = managed_test_handle(config, 128);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("object");
Download::orchestrate_to_path(
Arc::clone(&handle),
input,
dest.clone(),
None,
&ScriptedSinkFactory(Arc::clone(&script)),
)
.await
.unwrap()
.join()
.await
.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), &object[..]);
let events = script.events();
let written: usize = events
.iter()
.map(|event| match event {
Event::Finished(Op::Write { len, .. }, Ok(())) => *len,
_ => 0,
})
.sum();
assert_eq!(written, object.len(), "every byte went through the script");
assert_eq!(
events.last(),
Some(&Event::Finished(
Op::Finalize {
len: object.len() as u64
},
Ok(())
))
);
crate::runtime::test_util::shutdown(&handle).await;
}
#[test]
fn markers_order_against_operations() {
let script = Arc::new(WriteScript::default())
.on(Match::NthWrite(2), Action::Delay(Duration::from_millis(10)));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let writer = scripted_writer(&script, std::fs::File::create(&path).unwrap(), 4);
script.mark("start");
fill(&writer, 0, b"ab");
writer.drain(DrainMode::Eager).unwrap();
script.mark("between");
fill(&writer, 2, b"cd");
writer.finalize(4).unwrap();
script.mark("end");
let first = Op::Write { pos: 0, len: 2 };
let second = Op::Write { pos: 2, len: 2 };
let finalize = Op::Finalize { len: 4 };
assert_eq!(
script.events(),
[
Event::Mark("start"),
Event::Started(first),
Event::Finished(first, Ok(())),
Event::Mark("between"),
Event::Started(second),
Event::Finished(second, Ok(())),
Event::Started(finalize),
Event::Finished(finalize, Ok(())),
Event::Mark("end"),
]
);
assert_eq!(std::fs::read(&path).unwrap(), b"abcd");
}
}