use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use keel_core_api::{AttemptResult, ENVELOPE_VERSION, ErrorClass, Request};
use keel_journal::{
CacheKey, Clock, DiscoveryStore, FlowDescriptor, FlowId, FlowStatus, Journal, ManualClock,
NewFlow, ProcessId, SqliteJournal, StepKey, StepOutcome,
};
use keelrun_core::FlowDescriptor as FlowIdentity;
use keelrun_core::{Engine, FlowManager};
use serde_json::{Value, json};
use tempfile::TempDir;
const T0: i64 = 1_783_728_000_000;
fn cache_request(target: &str) -> Request {
Request {
v: ENVELOPE_VERSION,
target: target.to_owned(),
op: format!("GET {target}/item"),
idempotent: true,
args_hash: Some("args-1".to_owned()),
}
}
fn plain_request(target: &str) -> Request {
Request {
v: ENVELOPE_VERSION,
target: target.to_owned(),
op: format!("GET {target}/x"),
idempotent: true,
args_hash: None,
}
}
fn persistent_policy(target: &str, ttl: &str) -> Value {
json!({ "target": { target: { "cache": { "ttl": ttl, "scope": "persistent" } } } })
}
#[tokio::test(start_paused = true)]
async fn persistent_cache_round_trips_across_engines_sharing_a_journal() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("journal.db");
let clock = ManualClock::new(T0);
let policy = persistent_policy("api.catalog.internal", "5m");
let request = cache_request("api.catalog.internal");
let cached = json!({ "sku": "A-1", "price": 42 });
let mut a = Engine::new();
a.attach_journal(SqliteJournal::open(&path, clock.clone()).expect("open journal a"));
a.configure(&policy).expect("valid policy");
let out_a = a
.execute(&request, {
let cached = cached.clone();
async move |_attempt| AttemptResult::Ok {
payload: cached.clone(),
}
})
.await;
assert_eq!(out_a.result, "ok");
assert!(!out_a.from_cache, "engine A must miss the cold cache");
assert_eq!(out_a.attempts, 1);
let mut b = Engine::new();
b.attach_journal(SqliteJournal::open(&path, clock.clone()).expect("open journal b"));
b.configure(&policy).expect("valid policy");
let out_b = b
.execute(&request, async |_attempt| AttemptResult::Ok {
payload: json!("LIVE-CALL-SHOULD-NOT-RUN"),
})
.await;
assert!(
out_b.from_cache,
"engine B must serve from the shared journal"
);
assert_eq!(out_b.attempts, 0);
assert_eq!(out_b.payload, Some(cached));
}
#[tokio::test(start_paused = true)]
async fn persistent_cache_expires_against_the_journal_clock() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("journal.db");
let clock = ManualClock::new(T0);
let request = cache_request("api.catalog.internal");
let mut engine = Engine::new();
engine.attach_journal(SqliteJournal::open(&path, clock.clone()).unwrap());
engine
.configure(&persistent_policy("api.catalog.internal", "1m"))
.unwrap();
let out1 = engine
.execute(&request, async |_| AttemptResult::Ok {
payload: json!("v1"),
})
.await;
assert!(!out1.from_cache);
assert_eq!(out1.attempts, 1);
clock.advance(30_000);
let out2 = engine
.execute(&request, async |_| AttemptResult::Ok {
payload: json!("SENTINEL"),
})
.await;
assert!(out2.from_cache);
assert_eq!(out2.attempts, 0);
assert_eq!(out2.payload, Some(json!("v1")));
clock.advance(31_000); let out3 = engine
.execute(&request, async |_| AttemptResult::Ok {
payload: json!("v2"),
})
.await;
assert!(
!out3.from_cache,
"the entry must expire against the journal clock"
);
assert_eq!(out3.attempts, 1);
assert_eq!(out3.payload, Some(json!("v2")));
}
#[tokio::test(start_paused = true)]
async fn discovery_accumulates_one_row_per_call() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("discovery.db");
let clock = ManualClock::new(T0);
let mut engine = Engine::new();
engine.attach_discovery(DiscoveryStore::open(&path, clock.clone()).unwrap());
engine
.configure(&json!({
"target": { "api.x": {
"retry": { "attempts": 3, "schedule": "fixed(10ms)", "on": ["conn"] }
} }
}))
.unwrap();
let ok = engine
.execute(&plain_request("api.x"), async |_| AttemptResult::Ok {
payload: json!("ok"),
})
.await;
assert_eq!(ok.result, "ok");
assert_eq!(ok.attempts, 1);
let fail = engine
.execute(&plain_request("api.x"), async |_| AttemptResult::Error {
class: ErrorClass::Conn,
http_status: None,
retry_after_ms: None,
message: String::from("connection refused"),
original: None,
})
.await;
assert_eq!(fail.result, "error");
assert_eq!(fail.attempts, 3);
let reader = DiscoveryStore::open(&path, clock.clone()).unwrap();
let snapshot = reader.snapshot().unwrap();
assert_eq!(snapshot.len(), 1, "one target → one row");
let stats = &snapshot[0];
assert_eq!(stats.target, "api.x");
assert_eq!(stats.calls, 2);
assert_eq!(stats.successes, 1);
assert_eq!(stats.failures, 1);
assert_eq!(stats.attempts, 4, "1 (success) + 3 (exhausted)");
assert_eq!(stats.retries, 2, "0 + 2");
assert_eq!(stats.last_error_class, Some(ErrorClass::Conn));
}
#[tokio::test(start_paused = true)]
async fn unwritable_journal_degrades_persistent_cache_to_live_calls() {
let mut engine = Engine::new();
engine.attach_journal(FailingJournal);
engine
.configure(&persistent_policy("api.catalog.internal", "5m"))
.unwrap();
let warns = Arc::new(AtomicUsize::new(0));
let dispatch = tracing::Dispatch::new(WarnCounter(warns.clone()));
let guard = tracing::dispatcher::set_default(&dispatch);
let out = engine
.execute(&cache_request("api.catalog.internal"), async |_| {
AttemptResult::Ok {
payload: json!({ "ok": true }),
}
})
.await;
drop(guard);
assert_eq!(out.result, "ok");
assert!(!out.from_cache);
assert_eq!(out.attempts, 1);
assert_eq!(out.payload, Some(json!({ "ok": true })));
assert!(
warns.load(Ordering::SeqCst) >= 2,
"expected read + write degradation warnings, saw {}",
warns.load(Ordering::SeqCst)
);
}
#[tokio::test(start_paused = true)]
async fn policy_file_journal_is_attached_at_the_custom_path() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("custom").join("nested").join("j.db");
let mut policy = persistent_policy("api.catalog.internal", "5m");
policy["journal"] = json!(format!("file:{}", path.display()));
let a = Engine::new();
a.configure(&policy).expect("file: journal is honored");
assert!(a.journal().is_some(), "configure attached the journal");
assert!(
path.exists(),
"store created at the policy path, directories included"
);
let out_a = a
.execute(&cache_request("api.catalog.internal"), async |_| {
AttemptResult::Ok {
payload: json!("v1"),
}
})
.await;
assert!(!out_a.from_cache, "cold cache on the fresh custom store");
let b = Engine::new();
b.configure(&policy)
.expect("same policy on a second engine");
let out_b = b
.execute(&cache_request("api.catalog.internal"), async |_| {
AttemptResult::Ok {
payload: json!("SENTINEL-NOT-SERVED"),
}
})
.await;
assert!(
out_b.from_cache,
"the entry round-trips through the custom path"
);
assert_eq!(out_b.payload, Some(json!("v1")));
}
#[tokio::test(start_paused = true)]
async fn flows_land_in_the_policy_selected_journal() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("flows").join("j.db");
let engine = Arc::new(Engine::new());
engine
.configure(&json!({ "journal": format!("file:{}", path.display()) }))
.unwrap();
let journal = engine.journal().expect("policy attached a journal");
let clock: Arc<dyn Clock> = Arc::new(ManualClock::new(T0));
let manager = FlowManager::new(
Arc::clone(&engine),
Arc::clone(&journal),
clock,
ProcessId::new("host-a:pid-1"),
);
let mut handle = manager
.enter_flow(&FlowIdentity {
entrypoint: "py:pipeline.ingest:main".to_owned(),
args_hash: "a1".to_owned(),
explicit_key: None,
code_hash: Some("ch-1".to_owned()),
})
.unwrap();
let fid = handle.flow_id().clone();
let out = handle
.execute_step(&cache_request("api.step.internal"), |_attempt: u32| async {
AttemptResult::Ok {
payload: json!({ "rows": 1 }),
}
})
.await;
assert_eq!(out.result, "ok");
handle.complete_success().unwrap();
drop(handle);
let reader = SqliteJournal::open(&path, ManualClock::new(T0)).unwrap();
let flow = reader
.get_flow(&fid)
.unwrap()
.expect("flow journaled at the policy path");
assert_eq!(flow.status, FlowStatus::Completed);
}
#[tokio::test(start_paused = true)]
async fn malformed_postgres_journal_fails_configure_with_e040() {
let engine = Engine::new();
engine
.configure(&persistent_policy("api.catalog.internal", "5m"))
.unwrap();
let err = engine
.configure(&json!({ "journal": "postgres://keel:sekrit@[not-a-valid-host/keel" }))
.expect_err("a malformed postgres:// location cannot be opened");
assert_eq!(err.code.as_str(), "KEEL-E040");
assert!(
!err.message.contains("sekrit"),
"credentials never enter the diagnostic"
);
assert!(
engine.journal().is_none(),
"the rejected location attaches nothing"
);
let out1 = engine
.execute(&cache_request("api.catalog.internal"), async |_| {
AttemptResult::Ok { payload: json!(1) }
})
.await;
let out2 = engine
.execute(&cache_request("api.catalog.internal"), async |_| {
AttemptResult::Ok { payload: json!(2) }
})
.await;
assert!(!out1.from_cache);
assert!(
out2.from_cache,
"previous policy still in force after the rejected configure"
);
}
#[tokio::test(start_paused = true)]
async fn absent_journal_key_keeps_the_construction_attachment() {
let dir = TempDir::new().unwrap();
let mut engine = Engine::new();
engine.attach_journal(
SqliteJournal::open(dir.path().join("journal.db"), ManualClock::new(T0)).unwrap(),
);
let before = engine.journal().expect("attached at construction");
engine
.configure(&persistent_policy("api.catalog.internal", "5m"))
.unwrap();
let after = engine.journal().expect("still attached");
assert!(
Arc::ptr_eq(&before, &after),
"no journal key leaves the attachment untouched"
);
}
#[tokio::test(start_paused = true)]
async fn policy_journal_replaces_construction_attachment_and_reapplies_idempotently() {
let dir = TempDir::new().unwrap();
let mut engine = Engine::new();
engine.attach_journal(
SqliteJournal::open(dir.path().join("default.db"), ManualClock::new(T0)).unwrap(),
);
let constructed = engine.journal().unwrap();
let custom = dir.path().join("custom.db");
let mut policy = persistent_policy("api.catalog.internal", "5m");
policy["journal"] = json!(format!("file:{}", custom.display()));
engine.configure(&policy).unwrap();
let selected = engine.journal().unwrap();
assert!(
!Arc::ptr_eq(&constructed, &selected),
"policy file: replaces the construction journal"
);
assert!(custom.exists());
engine.configure(&policy).unwrap();
let reapplied = engine.journal().unwrap();
assert!(
Arc::ptr_eq(&selected, &reapplied),
"an unchanged location keeps the open store"
);
}
#[derive(Debug)]
struct FailingJournal;
fn injected_failure() -> keel_journal::Error {
keel_journal::Error::Corrupt {
column: "failing-journal",
value: "injected failure".to_owned(),
}
}
impl Journal for FailingJournal {
fn begin_flow(&self, _flow: &NewFlow) -> keel_journal::Result<FlowId> {
Err(injected_failure())
}
fn record_step(
&self,
_flow: &FlowId,
_seq: u64,
_key: &StepKey,
_outcome: &StepOutcome,
) -> keel_journal::Result<()> {
Err(injected_failure())
}
fn lookup_step(
&self,
_flow: &FlowId,
_seq: u64,
_key: &StepKey,
) -> keel_journal::Result<Option<StepOutcome>> {
Err(injected_failure())
}
fn step_at(
&self,
_flow: &FlowId,
_seq: u64,
) -> keel_journal::Result<Option<(StepKey, StepOutcome)>> {
Err(injected_failure())
}
fn get_flow(&self, _flow: &FlowId) -> keel_journal::Result<Option<FlowDescriptor>> {
Err(injected_failure())
}
fn complete_flow(&self, _flow: &FlowId, _status: FlowStatus) -> keel_journal::Result<()> {
Err(injected_failure())
}
fn incomplete_flows(&self, _lease_expired: bool) -> keel_journal::Result<Vec<FlowDescriptor>> {
Err(injected_failure())
}
fn acquire_lease(
&self,
_flow: &FlowId,
_holder: &ProcessId,
_ttl: Duration,
) -> keel_journal::Result<bool> {
Err(injected_failure())
}
fn put_cache(
&self,
_key: &CacheKey,
_value: &[u8],
_ttl: Duration,
) -> keel_journal::Result<()> {
Err(injected_failure())
}
fn get_cache(&self, _key: &CacheKey) -> keel_journal::Result<Option<Vec<u8>>> {
Err(injected_failure())
}
fn flows_by_entrypoint(&self, _entrypoint: &str) -> keel_journal::Result<Vec<FlowDescriptor>> {
Err(injected_failure())
}
fn steps_for_flow(&self, _flow: &FlowId) -> keel_journal::Result<Vec<(StepKey, StepOutcome)>> {
Err(injected_failure())
}
}
#[derive(Debug)]
struct WarnCounter(Arc<AtomicUsize>);
impl tracing::Subscriber for WarnCounter {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
if *event.metadata().level() == tracing::Level::WARN {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
fn enter(&self, _span: &tracing::span::Id) {}
fn exit(&self, _span: &tracing::span::Id) {}
}
#[derive(Debug)]
struct DebugCounter(Arc<AtomicUsize>);
impl tracing::Subscriber for DebugCounter {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
if *event.metadata().level() == tracing::Level::DEBUG {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
fn enter(&self, _span: &tracing::span::Id) {}
fn exit(&self, _span: &tracing::span::Id) {}
}
struct SelectiveFailJournal {
inner: SqliteJournal<ManualClock>,
fail_record_step: bool,
fail_complete_flow: bool,
}
impl Journal for SelectiveFailJournal {
fn begin_flow(&self, flow: &NewFlow) -> keel_journal::Result<FlowId> {
self.inner.begin_flow(flow)
}
fn record_step(
&self,
flow: &FlowId,
seq: u64,
key: &StepKey,
outcome: &StepOutcome,
) -> keel_journal::Result<()> {
if self.fail_record_step {
return Err(injected_failure());
}
self.inner.record_step(flow, seq, key, outcome)
}
fn lookup_step(
&self,
flow: &FlowId,
seq: u64,
key: &StepKey,
) -> keel_journal::Result<Option<StepOutcome>> {
self.inner.lookup_step(flow, seq, key)
}
fn step_at(
&self,
flow: &FlowId,
seq: u64,
) -> keel_journal::Result<Option<(StepKey, StepOutcome)>> {
self.inner.step_at(flow, seq)
}
fn get_flow(&self, flow: &FlowId) -> keel_journal::Result<Option<FlowDescriptor>> {
self.inner.get_flow(flow)
}
fn complete_flow(&self, flow: &FlowId, status: FlowStatus) -> keel_journal::Result<()> {
if self.fail_complete_flow {
return Err(injected_failure());
}
self.inner.complete_flow(flow, status)
}
fn incomplete_flows(&self, lease_expired: bool) -> keel_journal::Result<Vec<FlowDescriptor>> {
self.inner.incomplete_flows(lease_expired)
}
fn acquire_lease(
&self,
flow: &FlowId,
holder: &ProcessId,
ttl: Duration,
) -> keel_journal::Result<bool> {
self.inner.acquire_lease(flow, holder, ttl)
}
fn put_cache(&self, key: &CacheKey, value: &[u8], ttl: Duration) -> keel_journal::Result<()> {
self.inner.put_cache(key, value, ttl)
}
fn get_cache(&self, key: &CacheKey) -> keel_journal::Result<Option<Vec<u8>>> {
self.inner.get_cache(key)
}
fn flows_by_entrypoint(&self, entrypoint: &str) -> keel_journal::Result<Vec<FlowDescriptor>> {
self.inner.flows_by_entrypoint(entrypoint)
}
fn steps_for_flow(&self, flow: &FlowId) -> keel_journal::Result<Vec<(StepKey, StepOutcome)>> {
self.inner.steps_for_flow(flow)
}
}
fn attempt_marker_descriptor() -> FlowIdentity {
FlowIdentity {
entrypoint: "py:pipeline.ingest:main".to_owned(),
args_hash: "a1".to_owned(),
explicit_key: None,
code_hash: Some("ch-1".to_owned()),
}
}
#[tokio::test(start_paused = true)]
async fn flow_complete_propagates_journal_write_failure() {
let dir = TempDir::new().unwrap();
let clock = ManualClock::new(T0);
let inner = SqliteJournal::open(dir.path().join("journal.db"), clock.clone()).unwrap();
let journal: Arc<dyn Journal> = Arc::new(SelectiveFailJournal {
inner,
fail_record_step: false,
fail_complete_flow: true,
});
let engine = Arc::new(Engine::new());
let clock_dyn: Arc<dyn Clock> = Arc::new(clock);
let manager = FlowManager::new(engine, journal, clock_dyn, ProcessId::new("host-a:pid-1"));
let mut handle = manager.enter_flow(&attempt_marker_descriptor()).unwrap();
let err = handle
.complete_success()
.expect_err("a broken journal write must surface as Err, not vanish");
assert_eq!(err.code.as_str(), "KEEL-E040");
let mut handle2 = manager
.enter_flow(&FlowIdentity {
args_hash: "a2".to_owned(),
..attempt_marker_descriptor()
})
.unwrap();
let err2 = handle2
.complete_failed()
.expect_err("complete_failed must propagate the same way as complete_success");
assert_eq!(err2.code.as_str(), "KEEL-E040");
}
#[tokio::test(start_paused = true)]
async fn flow_enter_propagates_attempt_marker_write_failure() {
let dir = TempDir::new().unwrap();
let clock = ManualClock::new(T0);
let inner = SqliteJournal::open(dir.path().join("journal.db"), clock.clone()).unwrap();
let journal: Arc<dyn Journal> = Arc::new(SelectiveFailJournal {
inner,
fail_record_step: true,
fail_complete_flow: false,
});
let engine = Arc::new(Engine::new());
let clock_dyn: Arc<dyn Clock> = Arc::new(clock);
let manager = FlowManager::new(engine, journal, clock_dyn, ProcessId::new("host-a:pid-1"));
let err = manager
.enter_flow(&attempt_marker_descriptor())
.expect_err("a broken attempt-counter write must fail enter(), not vanish");
assert_eq!(err.code.as_str(), "KEEL-E040");
}
#[tokio::test(start_paused = true)]
async fn flow_complete_failure_still_marks_the_handle_completed_on_drop() {
let debugs = Arc::new(AtomicUsize::new(0));
let dispatch = tracing::Dispatch::new(DebugCounter(debugs.clone()));
let _guard = tracing::dispatcher::set_default(&dispatch);
let dir = TempDir::new().unwrap();
let clock = ManualClock::new(T0);
let inner = SqliteJournal::open(dir.path().join("journal.db"), clock.clone()).unwrap();
let journal: Arc<dyn Journal> = Arc::new(SelectiveFailJournal {
inner,
fail_record_step: false,
fail_complete_flow: true,
});
let engine = Arc::new(Engine::new());
let clock_dyn: Arc<dyn Clock> = Arc::new(clock);
let manager = FlowManager::new(engine, journal, clock_dyn, ProcessId::new("host-a:pid-1"));
let mut handle = manager.enter_flow(&attempt_marker_descriptor()).unwrap();
handle
.complete_success()
.expect_err("journal is configured to fail complete_flow");
drop(handle);
assert_eq!(
debugs.load(Ordering::SeqCst),
0,
"a failed complete() must still set self.completed so Drop does not \
also log the handle as crashed/uncompleted"
);
}