use core::time::Duration;
use super::{AckMode, Effect, Feeder, ItemId, Listen, NodeName, Station};
#[test]
fn an_item_id_is_its_sequence_position_and_orders_by_it() {
let first = ItemId::new(0);
let later = ItemId::new(9);
assert_eq!(first.seq(), 0);
assert!(first < later);
assert_eq!(later.to_string(), "item#9");
}
#[test]
fn the_four_delivery_modes_name_themselves() {
assert_eq!(Listen::Channel.as_str(), "channel");
assert_eq!(Listen::Changes(Feeder::InGraph).as_str(), "changes");
assert_eq!(Listen::Poll(Duration::from_secs(600)).as_str(), "poll");
assert_eq!(Listen::Transient.as_str(), "transient");
assert!(Listen::Channel.is_channel());
assert!(Listen::Poll(Duration::from_secs(600)).is_poll());
assert!(Listen::Transient.is_transient());
assert!(!Listen::Transient.is_channel());
assert_eq!(Listen::Changes(Feeder::InGraph).to_string(), "changes");
assert_eq!(Listen::Transient.to_string(), "transient");
assert_eq!(Listen::Poll(Duration::from_secs(1)).to_string(), "poll");
assert_ne!(
Listen::Poll(Duration::from_secs(1)),
Listen::Poll(Duration::from_secs(2)),
"the interval is part of the wiring even though it is not part of the name"
);
}
#[test]
fn the_two_ack_modes_name_themselves() {
assert_eq!(AckMode::Local.as_str(), "local");
assert_eq!(AckMode::Chained.as_str(), "chained");
assert!(AckMode::Chained.is_chained());
assert_eq!(AckMode::Local.to_string(), "local");
}
#[test]
fn the_two_effects_name_themselves() {
assert_eq!(Effect::Lands.as_str(), "lands");
assert_eq!(Effect::Pure.as_str(), "pure");
assert!(Effect::Pure.is_pure());
assert_eq!(Effect::Lands.to_string(), "lands");
}
#[test]
fn the_station_an_effect_reaches_is_the_one_its_lifecycle_carries() {
for effect in [Effect::Lands, Effect::Pure] {
assert_eq!(effect.reached(()).station(), effect.station());
}
assert_eq!(Effect::Lands.station(), Station::Persisted);
assert_eq!(Effect::Pure.station(), Station::Processed);
}
#[test]
fn a_node_name_is_a_borrowed_view_and_keys_a_map_as_its_string() {
use std::collections::HashMap;
let mut runs: HashMap<&NodeName, usize> = HashMap::new();
runs.insert(NodeName::new("decode"), 3);
assert_eq!(runs.get(NodeName::new("decode")), Some(&3));
assert_eq!(NodeName::new("decode").as_str(), "decode");
}
#[cfg(feature = "sqlite")]
mod durable {
use crate::{Context, drive::Drives};
use core::time::Duration;
use std::{
collections::BTreeMap,
path::Path,
sync::{Arc, Mutex},
thread,
time::Instant,
};
use sqlx::{Connection as _, Row as _, sqlite::SqliteConnection};
use super::super::{
ChangeIntake, Drained, Effect, ItemId, Resident, SqliteFeed, SqlitePositions, Sweeper, Track,
channel, doorbell,
};
use crate::{Delivery, Emitter, Operator};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct Scene {
pub(super) id: i64,
pub(super) title: String,
}
impl<'r> sqlx::FromRow<'r, sqlx::sqlite::SqliteRow> for Scene {
fn from_row(row: &'r sqlx::sqlite::SqliteRow) -> sqlx::Result<Self> {
Ok(Self {
id: row.try_get("id")?,
title: row.try_get("title")?,
})
}
}
type Titles = Arc<Mutex<BTreeMap<i64, String>>>;
struct Titling {
titles: Titles,
arrived: Option<flume::Sender<()>>,
gate: Option<flume::Receiver<()>>,
}
impl Titling {
fn new(titles: Titles) -> Self {
Self {
titles,
arrived: None,
gate: None,
}
}
fn gated(titles: Titles, arrived: flume::Sender<()>, gate: flume::Receiver<()>) -> Self {
Self {
titles,
arrived: Some(arrived),
gate: Some(gate),
}
}
}
impl Operator for Titling {
type Error = core::convert::Infallible;
type In = Scene;
type Out = Scene;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
_emitter: &mut Emitter<Scene>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let item = item.into_value();
if let Some(gate) = self.gate.take() {
if let Some(arrived) = self.arrived.take() {
arrived.send(()).expect("the walk is waiting");
}
gate
.recv_timeout(Duration::from_secs(20))
.expect("the gate opened");
}
self
.titles
.lock()
.expect("the node's own store")
.insert(item.id, item.title);
}
Ok(())
}
}
pub(super) async fn open(path: &Path) -> SqliteConnection {
let options = sqlx::sqlite::SqliteConnectOptions::new()
.filename(path)
.create_if_missing(true)
.busy_timeout(Duration::from_secs(10));
SqliteConnection::connect_with(&options)
.await
.expect("the database file")
}
pub(super) async fn append(path: &Path, rows: &[(i64, &str)]) {
let mut db = open(path).await;
sqlx::query("CREATE TABLE IF NOT EXISTS scene (id INTEGER PRIMARY KEY, title TEXT NOT NULL)")
.execute(&mut db)
.await
.expect("the table");
for (id, title) in rows {
sqlx::query("INSERT INTO scene (id, title) VALUES (?, ?)")
.bind(id)
.bind(title)
.execute(&mut db)
.await
.expect("the row");
}
}
pub(super) async fn durable(path: &Path, node: &str, source: &str) -> Option<i64> {
let mut watcher = open(path).await;
sqlx::query("SELECT position FROM ingraph_position WHERE unit = ? AND source = ?")
.bind(node)
.bind(source)
.fetch_optional(&mut watcher)
.await
.expect("the durable read")
.map(|row| row.get::<i64, _>(0))
}
pub(super) async fn position_rows(path: &Path) -> i64 {
let mut watcher = open(path).await;
sqlx::query("SELECT COUNT(*) FROM ingraph_position")
.fetch_one(&mut watcher)
.await
.expect("the count")
.get(0)
}
async fn process(path: &Path, titles: &Titles, until: i64) -> Drained {
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(path).await)
.await
.expect("the positions table");
let (out, inbox) = channel::<ItemId, Scene>(8);
let resident = Resident::new("titles", Titling::new(titles.clone()), inbox)
.with_effect(Effect::Lands)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _>::new(Track::new("titles", "scene"), feed, positions, out)
.with_page(2);
let deadline = Instant::now() + Duration::from_secs(20);
loop {
intake.drain().await.expect("the walk");
if intake.position() == Some(&until) {
break;
}
assert!(
Instant::now() < deadline,
"the position stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
engine.join().expect("the engine thread")
}
#[tokio::test]
async fn the_loop_carries_a_real_table_to_a_position_that_is_in_the_database() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two"), (3, "three")]).await;
let titles: Titles = Arc::new(Mutex::new(BTreeMap::new()));
let report = process(&path, &titles, 3).await;
assert_eq!(report.items(), 3);
assert_eq!(report.tally().processed(), 3);
assert_eq!(report.tally().emitted(), 0, "a landing node that emits");
assert_eq!(
titles
.lock()
.expect("the store")
.values()
.cloned()
.collect::<Vec<_>>(),
["one", "two", "three"]
);
assert_eq!(durable(&path, "titles", "scene").await, Some(3));
assert_eq!(position_rows(&path).await, 1);
}
#[tokio::test]
async fn a_restart_resumes_past_the_durable_position_and_re_reads_nothing() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two")]).await;
let first: Titles = Arc::new(Mutex::new(BTreeMap::new()));
process(&path, &first, 2).await;
assert_eq!(durable(&path, "titles", "scene").await, Some(2));
append(&path, &[(3, "three")]).await;
let second: Titles = Arc::new(Mutex::new(BTreeMap::new()));
let report = process(&path, &second, 3).await;
assert_eq!(report.items(), 1, "only the row written since");
assert_eq!(
second
.lock()
.expect("the store")
.keys()
.copied()
.collect::<Vec<_>>(),
[3]
);
assert_eq!(durable(&path, "titles", "scene").await, Some(3));
assert_eq!(position_rows(&path).await, 1);
}
#[tokio::test]
async fn two_units_over_one_table_keep_positions_that_do_not_collide() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two")]).await;
let titles: Titles = Arc::new(Mutex::new(BTreeMap::new()));
process(&path, &titles, 2).await;
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(&path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(&path).await)
.await
.expect("the positions table");
let (out, inbox) = channel::<ItemId, Scene>(8);
let other: Titles = Arc::new(Mutex::new(BTreeMap::new()));
let resident = Resident::new("captions", Titling::new(other.clone()), inbox)
.with_effect(Effect::Lands)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _>::new(
Track::new("captions", "scene"),
feed,
positions,
out,
);
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&2) {
intake.drain().await.expect("the second node's walk");
assert!(Instant::now() < deadline, "the second node stalled");
thread::sleep(Duration::from_millis(5));
}
drop(intake);
engine.join().expect("the engine thread");
assert_eq!(other.lock().expect("the store").len(), 2, "from the start");
assert_eq!(durable(&path, "titles", "scene").await, Some(2));
assert_eq!(durable(&path, "captions", "scene").await, Some(2));
assert_eq!(position_rows(&path).await, 2, "one row per track");
}
const QUIESCED: Duration = Duration::from_millis(200);
async fn pause(how_long: Duration) {
tokio::task::spawn_blocking(move || thread::sleep(how_long))
.await
.expect("the pause");
}
#[tokio::test]
async fn a_quiescent_store_needs_the_sweep_before_an_answer_becomes_durable() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two")]).await;
let titles: Titles = Arc::new(Mutex::new(BTreeMap::new()));
let (hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(&path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(&path).await)
.await
.expect("the positions table");
let (out, inbox) = channel::<ItemId, Scene>(8);
let (report_arrival, arrived) = flume::bounded::<()>(1);
let (open_gate, gate) = flume::bounded::<()>(1);
let resident = Resident::new(
"titles",
Titling::gated(titles.clone(), report_arrival, gate),
inbox,
)
.with_effect(Effect::Lands)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _>::new(Track::new("titles", "scene"), feed, positions, out);
let driving = tokio::spawn(async move {
let outcome = intake.run().await;
(outcome, intake)
});
let waiting =
tokio::task::spawn_blocking(move || arrived.recv_timeout(Duration::from_secs(20)));
waiting.await.expect("the wait").expect("the page arrived");
pause(QUIESCED).await;
assert_eq!(
durable(&path, "titles", "scene").await,
None,
"nothing has been answered yet"
);
open_gate.send(()).expect("the consumer is waiting");
pause(QUIESCED).await;
assert_eq!(
titles.lock().expect("the store").len(),
2,
"the consumer's own write is durable"
);
assert_eq!(
durable(&path, "titles", "scene").await,
None,
"an answer nobody collected is not a position"
);
let sweeper = Sweeper::spawn(Duration::from_millis(5), vec![Box::new(hint)]);
let deadline = Instant::now() + Duration::from_secs(20);
loop {
if durable(&path, "titles", "scene").await == Some(2) {
break;
}
assert!(Instant::now() < deadline, "the sweep never brought it back");
pause(Duration::from_millis(5)).await;
}
drop(sweeper);
let (outcome, intake) = driving.await.expect("the intake task");
outcome.expect("the intake ran to silence");
assert_eq!(intake.position(), Some(&2));
drop(intake);
engine.join().expect("the engine thread");
}
}
mod blob {
use crate::{Context, drive::Drives};
use core::time::Duration;
use std::{fs, path::PathBuf, thread};
use super::super::{
Answer, AsyncLands, Commit, Delivery, Item, ItemId, Payload, Resident, Station, ack_channel,
channel,
};
use crate::{
AsyncBlobEngine, AsyncBlobStore, AsyncLocate, BlobRecord, Connected, Emitter, Identifiable,
KeySegments, LocalFsBlob, LocalFsBlobError, LocalFsBlobRecord, LocalFsStore, Operator,
Segments, Transformer,
};
const SLOT: &str = "thumbnail";
#[derive(Debug, Clone, PartialEq, Eq)]
struct ThumbKey {
media: String,
frame: i64,
}
impl KeySegments for ThumbKey {
fn segments(&self, out: &mut Segments) {
out.push(&self.media);
out.push(&self.frame);
}
}
struct ThumbLineage {
#[expect(
dead_code,
reason = "`Transformer` names the type; no blob seam projects the lineage half"
)]
media: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Thumbnail {
key: ThumbKey,
bytes: Vec<u8>,
}
impl Identifiable for Thumbnail {
type Id = i64;
fn id_ref(&self) -> &i64 {
&self.key.frame
}
}
impl Transformer for Thumbnail {
type Graphql = ();
type Key = ThumbKey;
type Lineage = ThumbLineage;
type Record = Self;
fn key(&self) -> ThumbKey {
self.key.clone()
}
}
impl BlobRecord for Thumbnail {
type Bytes = Vec<u8>;
fn bytes(&self) -> &Vec<u8> {
&self.bytes
}
fn extension() -> Option<&'static str> {
Some("png")
}
}
struct BlobLane {
store: LocalFsStore,
handle: LocalFsBlob,
}
impl<R> AsyncLands<R> for BlobLane
where
R: LocalFsBlobRecord + Sync,
R::Bytes: AsRef<[u8]>,
{
type Error = LocalFsBlobError;
async fn land(&mut self, items: &[R]) -> Commit<Self::Error> {
let mut committed = Vec::new();
for (position, record) in items.iter().enumerate() {
match self
.handle
.put(Connected::new(&self.store, ()), record)
.await
{
Ok(()) => committed.push(position),
Err(error) => return Commit::only(committed, error),
}
}
Commit::all()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Frame {
media: String,
frame: i64,
bytes: Vec<u8>,
}
struct Thumbing;
impl Operator for Thumbing {
type Error = core::convert::Infallible;
type In = Frame;
type Out = Thumbnail;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Thumbnail>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let frame = item.into_value();
emitter.emit(Thumbnail {
key: ThumbKey {
media: frame.media,
frame: frame.frame,
},
bytes: frame.bytes,
});
}
Ok(())
}
}
fn located(store: &LocalFsStore, key: &ThumbKey) -> PathBuf {
let handle = LocalFsBlob::new(SLOT);
AsyncLocate::<Thumbnail>::locate(
&handle,
Connected::new(store, ()),
key,
Thumbnail::extension(),
)
.expect("the address")
}
#[test]
fn a_blob_unit_answers_after_its_bytes_are_on_disk() {
let dir = tempfile::tempdir().expect("a temporary directory");
let root = dir.path().join("blobs");
let store = LocalFsStore::open(&root).expect("the blob store");
let handle = store.blob(SLOT);
let (feed, inbox) = channel::<ItemId, Frame>(8);
let resident = Resident::new("thumbs", Thumbing, inbox)
.land(BlobLane {
store: store.clone(),
handle,
})
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let frames = [
Frame {
media: "movie".to_owned(),
frame: 1,
bytes: b"first".to_vec(),
},
Frame {
media: "movie".to_owned(),
frame: 2,
bytes: b"second".to_vec(),
},
];
let (ack, answers) = ack_channel();
let items = frames
.iter()
.zip(1u64..)
.map(|(frame, seq)| Item::new(ItemId::new(seq), frame.clone()))
.collect();
feed
.deliver(Delivery::new(Payload::Batch(items), ack))
.expect("the resident is listening");
let answer = loop {
match answers.take() {
Answer::Settled(settled) => break settled,
Answer::HungUp => panic!("the node hung up"),
Answer::Waiting => thread::sleep(Duration::from_millis(2)),
}
};
assert_eq!(answer.released().len(), 2);
assert_eq!(answer.observation().station(), Some(Station::Persisted));
for frame in &frames {
let key = ThumbKey {
media: frame.media.clone(),
frame: frame.frame,
};
let placed = located(&store, &key);
assert_eq!(
fs::read(&placed).expect("the placed blob"),
frame.bytes,
"at {placed:?}"
);
}
drop(feed);
let report = engine.join().expect("the engine thread");
assert_eq!(report.tally().refused(), 0);
assert_eq!(report.tally().emitted(), 0, "nothing went on a wire");
}
}
#[cfg(feature = "sqlite")]
mod socket {
use crate::{Context, drive::Drives};
use core::time::Duration;
use std::{path::Path, sync::Arc, thread, time::Instant};
use sqlx::{Row as _, sqlite::SqliteConnection};
use super::{
super::{
AckMode, AsyncLands, ChangeIntake, Commit, ItemId, RecordingTap, Resident, SqliteFeed,
SqlitePositions, Station, Taps, Track, channel, doorbell,
},
durable::{Scene, append, durable, open, position_rows},
};
use crate::{Delivery, Emitter, Operator};
#[derive(Debug, Clone, PartialEq, Eq)]
struct Title {
id: i64,
text: String,
}
struct Shouting;
impl Operator for Shouting {
type Error = core::convert::Infallible;
type In = Scene;
type Out = Title;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Title>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let scene = item.into_value();
emitter.emit(Title {
id: scene.id,
text: scene.title.to_uppercase(),
});
}
Ok(())
}
}
struct Keeping;
impl Operator for Keeping {
type Error = core::convert::Infallible;
type In = Title;
type Out = Title;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Title>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let title = item.into_value();
emitter.emit(title);
}
Ok(())
}
}
#[derive(Debug, derive_more::Display)]
#[display("the title table refused a row: {_0}")]
struct TitleRefused(sqlx::Error);
impl core::error::Error for TitleRefused {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
Some(&self.0)
}
}
struct TitleLane {
conn: SqliteConnection,
}
impl TitleLane {
async fn open(path: &Path) -> Self {
let mut conn = open(path).await;
sqlx::query("CREATE TABLE IF NOT EXISTS title (id INTEGER PRIMARY KEY, text TEXT NOT NULL)")
.execute(&mut conn)
.await
.expect("the title table");
Self { conn }
}
}
impl AsyncLands<Title> for TitleLane {
type Error = TitleRefused;
async fn land(&mut self, items: &[Title]) -> Commit<Self::Error> {
let mut committed = Vec::new();
for (position, title) in items.iter().enumerate() {
let written = sqlx::query(
"INSERT INTO title (id, text) VALUES (?, ?) \
ON CONFLICT (id) DO UPDATE SET text = excluded.text",
)
.bind(title.id)
.bind(title.text.as_str())
.execute(&mut self.conn)
.await;
match written {
Ok(_) => committed.push(position),
Err(error) => return Commit::only(committed, TitleRefused(error)),
}
}
Commit::all()
}
}
async fn titles(path: &Path) -> Vec<String> {
let mut watcher = open(path).await;
sqlx::query("SELECT text FROM title ORDER BY id")
.fetch_all(&mut watcher)
.await
.expect("the titles")
.iter()
.map(|row| row.get::<String, _>(0))
.collect()
}
async fn title_rows(path: &Path) -> i64 {
let mut watcher = open(path).await;
sqlx::query("SELECT COUNT(*) FROM title")
.fetch_one(&mut watcher)
.await
.expect("the count")
.get(0)
}
async fn until(deadline: Duration, what: &str, mut check: impl AsyncFnMut() -> bool) {
let give_up = Instant::now() + deadline;
while !check().await {
assert!(Instant::now() < give_up, "{what}");
thread::sleep(Duration::from_millis(5));
}
}
#[tokio::test]
async fn a_replay_after_a_lost_ack_rewrites_its_rows_and_the_position_lands_once() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two"), (3, "three")]).await;
{
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(&path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(&path).await)
.await
.expect("the positions table");
let (out, inbox) = channel::<ItemId, Scene>(8);
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _>::new(
Track::new("titles", "scene"),
feed,
positions,
out,
);
intake.drain().await.expect("the page went out");
assert_eq!(intake.position(), None, "nobody has answered anything");
let resident = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
until(
Duration::from_secs(20),
"the first write never landed",
async || title_rows(&path).await == 3,
)
.await;
assert_eq!(durable(&path, "titles", "scene").await, None);
drop(intake);
engine.join().expect("the engine thread");
}
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(&path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(&path).await)
.await
.expect("the positions table");
let (out, inbox) = channel::<ItemId, Scene>(8);
let resident = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _>::new(Track::new("titles", "scene"), feed, positions, out);
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&3) {
intake.drain().await.expect("the replay");
assert!(
Instant::now() < deadline,
"the replay stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
let report = engine.join().expect("the engine thread");
assert_eq!(report.items(), 3, "the whole source was read again");
assert_eq!(title_rows(&path).await, 3);
assert_eq!(titles(&path).await, ["ONE", "TWO", "THREE"]);
assert_eq!(durable(&path, "titles", "scene").await, Some(3));
assert_eq!(position_rows(&path).await, 1, "one row per track");
}
#[tokio::test]
async fn the_indexing_direction_runs_end_to_end_to_a_position_the_write_earned() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
append(&path, &[(1, "one"), (2, "two"), (3, "three")]).await;
let (to_store, store_inbox) = channel::<ItemId, Title>(8);
let storing = Resident::new("titles", Keeping, store_inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let store_engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let tap = Arc::new(RecordingTap::new());
let (to_pure, pure_inbox) = channel::<ItemId, Scene>(8);
let pure = Resident::new("shout", Shouting, pure_inbox)
.publish(to_store)
.with_ack(AckMode::Chained)
.with_taps(Taps::new().with(tap.clone()))
.with_sweep(Duration::from_millis(5));
let pure_engine = thread::spawn(move || {
let mut pure = pure;
pure.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Scene> =
SqliteFeed::new(open(&path).await, "scene", "id", bell).expect("the feed");
let positions = SqlitePositions::open(open(&path).await)
.await
.expect("the positions table");
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _>::new(
Track::new("shout", "scene"),
feed,
positions,
to_pure,
)
.with_page(2);
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&3) {
intake.drain().await.expect("the walk");
assert!(
Instant::now() < deadline,
"the chain stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
let pure_report = pure_engine.join().expect("the pure engine");
let store_report = store_engine.join().expect("the storing engine");
assert_eq!(titles(&path).await, ["ONE", "TWO", "THREE"]);
assert_eq!(title_rows(&path).await, 3);
assert_eq!(durable(&path, "shout", "scene").await, Some(3));
assert_eq!(position_rows(&path).await, 1);
assert_eq!(pure_report.tally().emitted(), 3);
assert_eq!(store_report.tally().processed(), 3);
assert_eq!(store_report.tally().refused(), 0);
assert!(
tap
.seen()
.iter()
.all(|(node, station)| node == "titles" && *station == Station::Persisted),
"the answers the outputs earned: {:?}",
tap.seen()
);
assert!(!tap.is_empty(), "the tap saw nothing at all");
}
}
#[cfg(feature = "sqlite")]
mod record {
use crate::{Context, drive::Drives};
use core::time::Duration;
use std::{
path::Path,
sync::{Arc, Mutex},
thread,
time::{Instant, SystemTime},
};
use sqlx::{Connection as _, Row as _, sqlite::SqliteConnection};
use super::{
super::{
AckMode, Answer, AsyncLands, ChangeIntake, Commit, Delivery, Item, ItemId, JournalRow,
JournalTap, JournalWriter, Ledger, NodeName, Payload, Positions, Resident, Station, Track,
ack_channel, channel, doorbell, journal_queue,
},
durable::open,
};
use crate::{Arrival, Attributes, Emitter, Operator, SqliteFeed};
#[derive(Debug, Clone, PartialEq, Eq)]
struct Shot {
id: i64,
media: i64,
title: String,
}
impl<'r> sqlx::FromRow<'r, sqlx::sqlite::SqliteRow> for Shot {
fn from_row(row: &'r sqlx::sqlite::SqliteRow) -> sqlx::Result<Self> {
Ok(Self {
id: row.try_get("id")?,
media: row.try_get("media")?,
title: row.try_get("title")?,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Title {
id: i64,
media: i64,
text: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Journal {
media: i64,
unit: String,
station: String,
}
impl JournalRow<i64> for Journal {
fn of(root: &i64, node: &NodeName, arrival: &Arrival, _at: SystemTime) -> Self {
Self {
media: *root,
unit: node.as_str().to_owned(),
station: arrival.station().as_str().to_owned(),
}
}
}
#[derive(Debug, derive_more::Display)]
#[display("the control store refused a statement: {_0}")]
struct ControlRefused(sqlx::Error);
impl core::error::Error for ControlRefused {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
Some(&self.0)
}
}
struct Control {
conn: SqliteConnection,
poison: bool,
}
impl Control {
async fn open(path: &Path) -> Self {
let mut conn = open(path).await;
for statement in [
"CREATE TABLE IF NOT EXISTS ingraph_position (\
unit TEXT NOT NULL, source TEXT NOT NULL, position INTEGER NOT NULL, \
PRIMARY KEY (unit, source)) WITHOUT ROWID",
"CREATE TABLE IF NOT EXISTS ingraph_journal (\
media INTEGER NOT NULL, unit TEXT NOT NULL, station TEXT NOT NULL, \
PRIMARY KEY (media, unit)) WITHOUT ROWID",
] {
sqlx::query(statement)
.execute(&mut conn)
.await
.expect("the control tables");
}
Self {
conn,
poison: false,
}
}
const fn poisoning(mut self) -> Self {
self.poison = true;
self
}
}
async fn write_journal(
tx: &mut sqlx::SqliteConnection,
rows: &[Journal],
) -> Result<(), ControlRefused> {
for row in rows {
sqlx::query(
"INSERT INTO ingraph_journal (media, unit, station) VALUES (?, ?, ?) \
ON CONFLICT (media, unit) DO UPDATE SET station = excluded.station",
)
.bind(row.media)
.bind(row.unit.as_str())
.bind(row.station.as_str())
.execute(&mut *tx)
.await
.map_err(ControlRefused)?;
}
Ok(())
}
impl Ledger for Control {
type Error = ControlRefused;
type Row = Journal;
fn record(&mut self, rows: &[Journal]) -> Result<(), ControlRefused> {
crate::drive::lane::block_on(self.record_async(rows))
}
async fn record_async(&mut self, rows: &[Journal]) -> Result<(), ControlRefused> {
if rows.is_empty() {
return Ok(());
}
let mut tx = self.conn.begin().await.map_err(ControlRefused)?;
write_journal(&mut tx, rows).await?;
tx.commit().await.map_err(ControlRefused)
}
}
impl Positions<i64> for Control {
fn position(&mut self, track: &Track) -> Result<Option<i64>, ControlRefused> {
crate::drive::lane::block_on(self.position_async(track))
}
fn advance(&mut self, track: &Track, to: i64, rows: &[Journal]) -> Result<(), ControlRefused> {
crate::drive::lane::block_on(self.advance_async(track, to, rows))
}
async fn position_async(&mut self, track: &Track) -> Result<Option<i64>, ControlRefused> {
let row = sqlx::query("SELECT position FROM ingraph_position WHERE unit = ? AND source = ?")
.bind(track.node())
.bind(track.source())
.fetch_optional(&mut self.conn)
.await
.map_err(ControlRefused)?;
row
.map(|row| row.try_get::<i64, _>(0))
.transpose()
.map_err(ControlRefused)
}
async fn advance_async(
&mut self,
track: &Track,
to: i64,
rows: &[Journal],
) -> Result<(), ControlRefused> {
let poison = self.poison;
let mut tx = self.conn.begin().await.map_err(ControlRefused)?;
write_journal(&mut tx, rows).await?;
let position = if poison {
sqlx::query("INSERT INTO ingraph_position (unit, source) VALUES (?, ?)")
.bind(track.node())
.bind(track.source())
.execute(&mut *tx)
.await
} else {
sqlx::query(
"INSERT INTO ingraph_position (unit, source, position) VALUES (?, ?, ?) \
ON CONFLICT (unit, source) DO UPDATE SET position = excluded.position",
)
.bind(track.node())
.bind(track.source())
.bind(to)
.execute(&mut *tx)
.await
};
position.map_err(ControlRefused)?;
tx.commit().await.map_err(ControlRefused)
}
}
struct Shouting;
impl Operator for Shouting {
type Error = core::convert::Infallible;
type In = Shot;
type Out = Title;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Title>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let shot = item.into_value();
emitter.emit(Title {
id: shot.id,
media: shot.media,
text: shot.title.to_uppercase(),
});
}
Ok(())
}
}
struct Keeping;
impl Operator for Keeping {
type Error = core::convert::Infallible;
type In = Title;
type Out = Title;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Title>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let title = item.into_value();
emitter.emit(title);
}
Ok(())
}
}
#[derive(Debug, derive_more::Display)]
#[display("the title table refused a row: {_0}")]
struct TitleRefused(sqlx::Error);
impl core::error::Error for TitleRefused {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
Some(&self.0)
}
}
struct TitleLane {
conn: SqliteConnection,
}
impl TitleLane {
async fn open(path: &Path) -> Self {
let mut conn = open(path).await;
sqlx::query(
"CREATE TABLE IF NOT EXISTS title (\
id INTEGER PRIMARY KEY, media INTEGER NOT NULL, text TEXT NOT NULL)",
)
.execute(&mut conn)
.await
.expect("the title table");
Self { conn }
}
}
impl AsyncLands<Title> for TitleLane {
type Error = TitleRefused;
async fn land(&mut self, items: &[Title]) -> Commit<Self::Error> {
let mut committed = Vec::new();
for (position, title) in items.iter().enumerate() {
let written = sqlx::query(
"INSERT INTO title (id, media, text) VALUES (?, ?, ?) \
ON CONFLICT (id) DO UPDATE SET text = excluded.text",
)
.bind(title.id)
.bind(title.media)
.bind(title.text.as_str())
.execute(&mut self.conn)
.await;
match written {
Ok(_) => committed.push(position),
Err(error) => return Commit::only(committed, TitleRefused(error)),
}
}
Commit::all()
}
}
struct Media {
seen: Arc<Mutex<Vec<(i64, String, Station)>>>,
}
impl Attributes<Shot> for Media {
type Root = i64;
fn root(&self, shot: &Shot) -> i64 {
shot.media
}
fn observe(&self, root: &i64, worker: &NodeName, arrival: &Arrival) {
self.seen.lock().expect("the seen list").push((
*root,
worker.as_str().to_owned(),
arrival.station(),
));
}
}
impl Attributes<Title> for Media {
type Root = i64;
fn root(&self, title: &Title) -> i64 {
title.media
}
fn observe(&self, root: &i64, worker: &NodeName, arrival: &Arrival) {
self.seen.lock().expect("the seen list").push((
*root,
worker.as_str().to_owned(),
arrival.station(),
));
}
}
async fn append(path: &Path, rows: &[(i64, i64, &str)]) {
let mut db = open(path).await;
sqlx::query(
"CREATE TABLE IF NOT EXISTS shot (\
id INTEGER PRIMARY KEY, media INTEGER NOT NULL, title TEXT NOT NULL)",
)
.execute(&mut db)
.await
.expect("the table");
for (id, media, title) in rows {
sqlx::query("INSERT INTO shot (id, media, title) VALUES (?, ?, ?)")
.bind(id)
.bind(media)
.bind(title)
.execute(&mut db)
.await
.expect("the row");
}
}
async fn journal_of(path: &Path) -> Vec<(i64, String, String)> {
let mut watcher = open(path).await;
sqlx::query("SELECT media, unit, station FROM ingraph_journal ORDER BY media, unit")
.fetch_all(&mut watcher)
.await
.expect("the journal rows")
.iter()
.map(|row| {
(
row.get::<i64, _>(0),
row.get::<String, _>(1),
row.get::<String, _>(2),
)
})
.collect()
}
async fn position(path: &Path, node: &str, source: &str) -> Option<i64> {
let mut watcher = open(path).await;
sqlx::query("SELECT position FROM ingraph_position WHERE unit = ? AND source = ?")
.bind(node)
.bind(source)
.fetch_optional(&mut watcher)
.await
.expect("the durable read")
.map(|row| row.get::<i64, _>(0))
}
async fn journal_rows(path: &Path) -> i64 {
let mut watcher = open(path).await;
sqlx::query("SELECT COUNT(*) FROM ingraph_journal")
.fetch_one(&mut watcher)
.await
.expect("the count")
.get(0)
}
async fn title_rows(path: &Path) -> i64 {
let mut watcher = open(path).await;
sqlx::query("SELECT COUNT(*) FROM title")
.fetch_one(&mut watcher)
.await
.expect("the count")
.get(0)
}
#[tokio::test]
async fn the_position_and_the_run_record_move_together_or_not_at_all() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(&path, &[(1, 100, "one"), (2, 100, "two")]).await;
let seen = Arc::new(Mutex::new(Vec::new()));
let (out, inbox) = channel::<ItemId, Shot>(8);
let storing = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(&path).await, "shot", "id", bell).expect("the feed");
let control = Control::open(&path).await.poisoning();
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _, _>::new(Track::new("titles", "shot"), feed, control, out)
.attributing(Media { seen });
let deadline = Instant::now() + Duration::from_secs(20);
let refusal = loop {
match intake.drain().await {
Err(refusal) => break refusal,
Ok(_) => {
assert!(Instant::now() < deadline, "the control never refused");
thread::sleep(Duration::from_millis(5));
}
}
};
assert!(refusal.is_positions(), "{refusal}");
assert_eq!(title_rows(&path).await, 2, "the content is durable");
assert_eq!(position(&path, "titles", "shot").await, None);
assert_eq!(journal_of(&path).await, [], "no row without the position");
drop(intake);
engine.join().expect("the engine thread");
}
#[tokio::test]
async fn a_healthy_control_lands_the_position_and_the_row_in_the_same_pass() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(
&path,
&[(1, 100, "one"), (2, 100, "two"), (3, 200, "three")],
)
.await;
let seen = Arc::new(Mutex::new(Vec::new()));
let (out, inbox) = channel::<ItemId, Shot>(8);
let storing = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(&path).await, "shot", "id", bell).expect("the feed");
let control = Control::open(&path).await;
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _, _>::new(Track::new("titles", "shot"), feed, control, out)
.attributing(Media { seen: seen.clone() });
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&3) {
intake.drain().await.expect("the walk");
assert!(
Instant::now() < deadline,
"stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
engine.join().expect("the engine thread");
assert_eq!(position(&path, "titles", "shot").await, Some(3));
assert_eq!(
journal_of(&path).await,
[
(100, "titles".to_owned(), "persisted".to_owned()),
(200, "titles".to_owned(), "persisted".to_owned()),
]
);
assert_eq!(title_rows(&path).await, 3);
}
async fn process(path: &Path, until: i64) {
let seen = Arc::new(Mutex::new(Vec::new()));
let (out, inbox) = channel::<ItemId, Shot>(8);
let storing = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(path).await, "shot", "id", bell).expect("the feed");
let control = Control::open(path).await;
let mut intake =
ChangeIntake::<sqlx::Sqlite, _, _, _>::new(Track::new("titles", "shot"), feed, control, out)
.with_page(2)
.attributing(Media { seen });
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&until) {
intake.drain().await.expect("the walk");
assert!(
Instant::now() < deadline,
"stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
engine.join().expect("the engine thread");
}
#[tokio::test]
async fn a_second_run_rewrites_the_row_rather_than_growing_the_table() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(
&path,
&[(1, 100, "one"), (2, 100, "two"), (3, 200, "three")],
)
.await;
process(&path, 3).await;
assert_eq!(journal_rows(&path).await, 2, "two medias, one node");
assert_eq!(position(&path, "titles", "shot").await, Some(3));
append(&path, &[(4, 100, "four"), (5, 200, "five")]).await;
process(&path, 5).await;
assert_eq!(journal_rows(&path).await, 2, "the same two rows, rewritten");
assert_eq!(
journal_of(&path).await,
[
(100, "titles".to_owned(), "persisted".to_owned()),
(200, "titles".to_owned(), "persisted".to_owned()),
]
);
assert_eq!(title_rows(&path).await, 5);
assert_eq!(position(&path, "titles", "shot").await, Some(5));
}
#[tokio::test]
async fn a_channel_fed_unit_writes_its_completion_row_with_a_lone_upsert() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
let seen = Arc::new(Mutex::new(Vec::new()));
let (feed, inbox) = channel::<ItemId, Title>(8);
let resident = Resident::new("titles", Keeping, inbox)
.land(TitleLane::open(&path).await)
.attributing(Media { seen: seen.clone() })
.recording_to(Control::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let (ack, answers) = ack_channel();
let items = [
Title {
id: 1,
media: 100,
text: "ONE".to_owned(),
},
Title {
id: 2,
media: 100,
text: "TWO".to_owned(),
},
Title {
id: 3,
media: 200,
text: "THREE".to_owned(),
},
]
.into_iter()
.zip(1u64..)
.map(|(title, seq)| Item::new(ItemId::new(seq), title))
.collect();
feed
.deliver(Delivery::new(Payload::Batch(items), ack))
.expect("the resident is listening");
let answer = loop {
match answers.take() {
Answer::Settled(settled) => break settled,
Answer::HungUp => panic!("the node hung up"),
Answer::Waiting => thread::sleep(Duration::from_millis(2)),
}
};
assert_eq!(answer.observation().station(), Some(Station::Persisted));
assert_eq!(answer.released().len(), 3);
assert_eq!(answer.worker().as_str(), "titles");
drop(feed);
engine.join().expect("the engine thread");
assert_eq!(
journal_of(&path).await,
[
(100, "titles".to_owned(), "persisted".to_owned()),
(200, "titles".to_owned(), "persisted".to_owned()),
]
);
assert_eq!(
position(&path, "titles", "shot").await,
None,
"a channel-fed node keeps no cursor"
);
assert_eq!(title_rows(&path).await, 3);
}
struct JournalLane {
conn: SqliteConnection,
}
impl AsyncLands<Journal> for JournalLane {
type Error = ControlRefused;
async fn land(&mut self, items: &[Journal]) -> Commit<Self::Error> {
let mut committed = Vec::new();
for (position, row) in items.iter().enumerate() {
let written = sqlx::query(
"INSERT INTO ingraph_journal (media, unit, station) VALUES (?, ?, ?) \
ON CONFLICT (media, unit) DO UPDATE SET station = excluded.station",
)
.bind(row.media)
.bind(row.unit.as_str())
.bind(row.station.as_str())
.execute(&mut self.conn)
.await;
match written {
Ok(_) => committed.push(position),
Err(error) => return Commit::only(committed, ControlRefused(error)),
}
}
Commit::all()
}
}
struct Shaky {
broken: i64,
}
#[derive(Debug, derive_more::Display)]
#[display("the operator cannot read media {_0}")]
struct Unreadable(i64);
impl core::error::Error for Unreadable {}
impl Operator for Shaky {
type Error = Unreadable;
type In = Shot;
type Out = Title;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Title>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let shot = item.into_value();
if shot.media == self.broken {
return Err(Unreadable(shot.media));
}
emitter.emit(Title {
id: shot.id,
media: shot.media,
text: shot.title.to_uppercase(),
});
}
Ok(())
}
}
#[tokio::test]
async fn the_indexing_direction_writes_a_run_record_through_both_of_its_halves() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(
&path,
&[(1, 100, "one"), (2, 100, "two"), (3, 200, "three")],
)
.await;
drop(Control::open(&path).await);
let (queue, drain) = journal_queue::<Journal>(64);
let mut writer = JournalWriter::new(
drain,
JournalLane {
conn: open(&path).await,
},
)
.with_patience(Duration::from_millis(5));
let scribe = thread::spawn(move || writer.run());
let (to_store, store_inbox) = channel::<ItemId, Title>(8);
let storing = Resident::new("titles", Keeping, store_inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let store_engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (to_pure, pure_inbox) = channel::<ItemId, Shot>(8);
let pure = Resident::new("shout", Shouting, pure_inbox)
.publish(to_store)
.with_ack(AckMode::Chained)
.attributing(JournalTap::<Title, i64, Journal>::new(
|title| title.media,
queue.clone(),
))
.with_sweep(Duration::from_millis(5));
let pure_engine = thread::spawn(move || {
let mut pure = pure;
pure.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(&path).await, "shot", "id", bell).expect("the feed");
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _, _>::new(
Track::new("shout", "shot"),
feed,
Control::open(&path).await,
to_pure,
)
.with_page(2)
.attributing(JournalTap::<Shot, i64, Journal>::new(
|shot| shot.media,
queue.clone(),
));
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&3) {
intake.drain().await.expect("the walk");
assert!(
Instant::now() < deadline,
"the chain stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
pure_engine.join().expect("the pure engine");
store_engine.join().expect("the storing engine");
drop(queue);
let scribed = scribe.join().expect("the writer thread");
assert!(scribed.written() > 0, "the diagnostic half wrote nothing");
assert_eq!(title_rows(&path).await, 3);
assert_eq!(position(&path, "shout", "shot").await, Some(3));
assert_eq!(journal_rows(&path).await, 4);
assert_eq!(
journal_of(&path).await,
[
(100, "shout".to_owned(), "processed".to_owned()),
(100, "titles".to_owned(), "persisted".to_owned()),
(200, "shout".to_owned(), "processed".to_owned()),
(200, "titles".to_owned(), "persisted".to_owned()),
]
);
}
#[tokio::test]
async fn a_failure_at_the_pure_unit_lands_as_the_badge_red_row() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(&path, &[(1, 100, "one"), (2, 200, "two")]).await;
drop(Control::open(&path).await);
let (queue, drain) = journal_queue::<Journal>(64);
let mut writer = JournalWriter::new(
drain,
JournalLane {
conn: open(&path).await,
},
)
.with_patience(Duration::from_millis(5));
let scribe = thread::spawn(move || writer.run());
let (to_store, store_inbox) = channel::<ItemId, Title>(8);
let storing = Resident::new("titles", Keeping, store_inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let store_engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (to_pure, pure_inbox) = channel::<ItemId, Shot>(8);
let pure = Resident::new("shout", Shaky { broken: 200 }, pure_inbox)
.publish(to_store)
.with_sweep(Duration::from_millis(5));
let pure_engine = thread::spawn(move || {
let mut pure = pure;
pure.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(&path).await, "shot", "id", bell).expect("the feed");
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _, _>::new(
Track::new("shout", "shot"),
feed,
Control::open(&path).await,
to_pure,
)
.attributing(JournalTap::<Shot, i64, Journal>::new(
|shot| shot.media,
queue.clone(),
));
let deadline = Instant::now() + Duration::from_secs(20);
loop {
intake.drain().await.expect("the walk");
if journal_of(&path)
.await
.iter()
.any(|(media, node, station)| *media == 200 && node == "shout" && station == "failed")
{
break;
}
assert!(Instant::now() < deadline, "the failure never reached a row");
thread::sleep(Duration::from_millis(5));
}
drop(intake);
pure_engine.join().expect("the pure engine");
store_engine.join().expect("the storing engine");
drop(queue);
scribe.join().expect("the writer thread");
let rows = journal_of(&path).await;
assert!(
rows.contains(&(200, "shout".to_owned(), "failed".to_owned())),
"the badge-red row: {rows:?}"
);
assert_eq!(title_rows(&path).await, 1);
}
#[tokio::test]
async fn a_run_record_with_no_writer_at_all_still_has_every_final_state() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("shots.db");
append(&path, &[(1, 100, "one"), (2, 200, "two")]).await;
let (queue, drain) = journal_queue::<Journal>(4);
drop(drain);
let (out, inbox) = channel::<ItemId, Shot>(8);
let storing = Resident::new("titles", Shouting, inbox)
.land(TitleLane::open(&path).await)
.with_sweep(Duration::from_millis(5));
let engine = thread::spawn(move || {
let mut storing = storing;
storing.run()
});
let (_hint, bell) = doorbell();
let feed: SqliteFeed<Shot> =
SqliteFeed::new(open(&path).await, "shot", "id", bell).expect("the feed");
let mut intake = ChangeIntake::<sqlx::Sqlite, _, _, _>::new(
Track::new("titles", "shot"),
feed,
Control::open(&path).await,
out,
)
.attributing(JournalTap::<Shot, i64, Journal>::new(
|shot| shot.media,
queue,
));
let deadline = Instant::now() + Duration::from_secs(20);
while intake.position() != Some(&2) {
intake.drain().await.expect("the walk");
assert!(
Instant::now() < deadline,
"stalled at {:?}",
intake.position()
);
thread::sleep(Duration::from_millis(5));
}
drop(intake);
engine.join().expect("the engine thread");
assert_eq!(
journal_of(&path).await,
[
(100, "titles".to_owned(), "persisted".to_owned()),
(200, "titles".to_owned(), "persisted".to_owned()),
],
"every final state, with no diagnostic writer in the process"
);
assert_eq!(position(&path, "titles", "shot").await, Some(2));
}
}
#[cfg(feature = "sqlite")]
mod reaper {
use crate::{Context, drive::Drives};
use core::time::Duration;
use std::{
path::Path,
sync::{Arc, Mutex},
thread,
time::Instant,
};
use sqlx::{Row as _, sqlite::SqliteConnection};
use super::{
super::{
AckMode, Drained, Enablement, ItemId, Listen, Plan, PollUpstream, RecordingTap, Resident,
Schedule, Seats, Station, Swept, Taps, UpstreamWiring, channel,
},
durable::{Scene, open},
};
use crate::{
ChannelGraph, Delivery, DriverKind, EdgeId, Emitter, FanoutKind, GraphBuilder, NodeId, Operator,
};
struct Lent<'a> {
slot: &'a Mutex<Option<SqliteConnection>>,
conn: Option<SqliteConnection>,
}
impl core::ops::Deref for Lent<'_> {
type Target = SqliteConnection;
fn deref(&self) -> &SqliteConnection {
self.conn.as_ref().expect("lent for the call")
}
}
impl core::ops::DerefMut for Lent<'_> {
fn deref_mut(&mut self) -> &mut SqliteConnection {
self.conn.as_mut().expect("lent for the call")
}
}
impl Drop for Lent<'_> {
fn drop(&mut self) {
*self.slot.lock().expect("the context's slot") = self.conn.take();
}
}
struct Granted {
expired: Mutex<Option<SqliteConnection>>,
census: Mutex<Option<SqliteConnection>>,
}
impl Granted {
async fn open(path: &Path) -> Self {
Self {
expired: Mutex::new(Some(open(path).await)),
census: Mutex::new(Some(open(path).await)),
}
}
fn expired(&self) -> Lent<'_> {
Lent {
slot: &self.expired,
conn: self.expired.lock().expect("the write slot").take(),
}
}
fn census(&self) -> Lent<'_> {
Lent {
slot: &self.census,
conn: self.census.lock().expect("the read slot").take(),
}
}
}
#[derive(Debug, derive_more::Display)]
#[display("the reaper's statement failed: {_0}")]
struct Unreachable(sqlx::Error);
impl core::error::Error for Unreachable {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
Some(&self.0)
}
}
struct Reaping {
looked: Arc<Mutex<Vec<PollUpstream>>>,
counted: Arc<Mutex<Vec<i64>>>,
}
const REAP: PollUpstream = PollUpstream::new(0);
const COUNT: PollUpstream = PollUpstream::new(1);
impl Operator for Reaping {
type Error = core::convert::Infallible;
type In = Scene;
type Out = Scene;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<Self::In>,
emitter: &mut Emitter<Scene>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let scene = item.into_value();
emitter.emit(scene);
}
Ok(())
}
}
impl crate::Declared for Reaping {
const EFFECT: crate::Effect = crate::Effect::Pure;
type Channels = crate::Unchanneled;
type Mode = crate::Block;
type Options = ();
type Pair = crate::Buffered;
type Wires<O> = crate::Downstream<O>;
type Attribution<O> = ();
}
impl Seats<Reaping> for Granted {
type Error = Unreachable;
type Info = u64;
async fn seat(
&mut self,
operator: &mut Reaping,
source: PollUpstream,
) -> Result<Swept<u64>, Unreachable> {
operator
.looked
.lock()
.expect("the walk's record")
.push(source);
if source == REAP {
let mut conn = self.expired();
let done = sqlx::query("DELETE FROM scene WHERE title = ?")
.bind("expired")
.execute(&mut *conn)
.await
.map_err(Unreachable)?;
return Ok(Swept::Ran(done.rows_affected()));
}
let mut conn = self.census();
let count: i64 = sqlx::query("SELECT COUNT(*) FROM scene")
.fetch_one(&mut *conn)
.await
.map_err(Unreachable)?
.get(0);
operator
.counted
.lock()
.expect("the walk's record")
.push(count);
Ok(Swept::Ran(0))
}
}
fn arrangement() -> ChannelGraph {
let mut builder = GraphBuilder::new();
let scenes = builder.source::<Reaping>("scene", DriverKind::Thread);
let reaper = builder.node::<Reaping>("reaper", DriverKind::Thread);
builder.edge(&scenes, &reaper, FanoutKind::Broadcast);
builder.validate().expect("one edge validates")
}
async fn titles(path: &Path) -> Vec<String> {
let mut watcher = open(path).await;
sqlx::query("SELECT title FROM scene ORDER BY id")
.fetch_all(&mut watcher)
.await
.expect("the titles")
.iter()
.map(|row| row.get::<String, _>(0))
.collect()
}
#[tokio::test]
async fn a_reaper_over_a_disabled_upstream_runs_two_clocks_and_only_one_of_them_writes() {
let dir = tempfile::tempdir().expect("a temporary directory");
let path = dir.path().join("scenes.db");
super::durable::append(
&path,
&[
(1, "keep"),
(2, "expired"),
(3, "expired"),
(4, "keep"),
(5, "expired"),
],
)
.await;
let wired = Plan::new(arrangement())
.upstream(
EdgeId::new(0),
UpstreamWiring::new(Listen::Poll(Duration::from_millis(5)), AckMode::Local),
)
.with_enablement(Enablement::new().launching("reaper"))
.validate()
.expect("reaping does not need the dead alive");
assert!(wired.is_enabled(NodeId::new(1)), "the reaper runs");
assert!(!wired.is_enabled(NodeId::new(0)), "its upstream does not");
let looked = Arc::new(Mutex::new(Vec::new()));
let counted = Arc::new(Mutex::new(Vec::new()));
let tap = Arc::new(RecordingTap::new());
let (out, inbox) = channel::<ItemId, Scene>(8);
let resident = Resident::new(
"reaper",
Reaping {
looked: looked.clone(),
counted: counted.clone(),
},
inbox,
)
.polling(
Schedule::new()
.every(Duration::from_millis(5))
.every(Duration::from_millis(120)),
Granted::open(&path).await,
)
.with_taps(Taps::new().with(tap.clone()))
.with_sweep(Duration::from_secs(30));
let engine = thread::spawn(move || {
let mut resident = resident;
resident.run()
});
let looks = |source: PollUpstream| {
looked
.lock()
.expect("the walk's record")
.iter()
.filter(|&&seen| seen == source)
.count()
};
let deadline = Instant::now() + Duration::from_secs(30);
while looks(REAP) < 12 || looks(COUNT) < 1 {
assert!(
Instant::now() < deadline,
"the two clocks never both ran: {} and {}",
looks(REAP),
looks(COUNT)
);
thread::sleep(Duration::from_millis(5));
}
drop(out);
let report: Drained = engine.join().expect("the engine thread");
let fast = looks(REAP);
let slow = looks(COUNT);
assert!(
fast > slow,
"each source keeps its own cadence: {fast} and {slow}"
);
assert_eq!(
report.ticks(),
fast + slow,
"one look per upstream per pass"
);
assert_eq!(report.deliveries(), 0, "nothing was ever fed to it");
assert_eq!(titles(&path).await, ["keep", "keep"]);
let census = counted.lock().expect("the walk's record").clone();
assert!(!census.is_empty(), "the read-only slot was never used");
assert!(census.iter().all(|&count| count == 2), "{census:?}");
assert!(
tap
.seen()
.iter()
.all(|(node, station)| node == "reaper" && *station == Station::Processed),
"{:?}",
tap.seen()
);
assert_eq!(tap.len(), fast + slow, "every look was observed");
}
}