use std::future::Future;
use std::pin::pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::task::{Context, Poll, Waker};
use keel_core_api::{AttemptResult, ENVELOPE_VERSION, Request};
use keel_journal::{Clock, Journal, ManualClock, ProcessId, SqliteJournal, StepStatus};
use keelrun_core::{Engine, FlowDescriptor, FlowManager};
use serde_json::{Value, json};
use tempfile::TempDir;
const T0: i64 = 1_783_728_000_000;
fn request(target: &str, args_hash: &str) -> Request {
Request {
v: ENVELOPE_VERSION,
target: target.to_owned(),
op: format!("POST {target}/charges"),
idempotent: true, args_hash: Some(args_hash.to_owned()),
}
}
#[test]
fn engine_resolves_the_idempotency_header_per_target() {
let engine = Engine::new();
engine
.configure(&json!({
"defaults": { "outbound": { "idempotency": { "header": "X-Idem" } } },
"target": {
"api.stripe.com": { "idempotency": { "header": "Idempotency-Key" } },
"api.nokey.example": { "timeout": "1s" }
}
}))
.expect("valid policy");
assert_eq!(
engine.idempotency_header("api.stripe.com").as_deref(),
Some("Idempotency-Key")
);
assert_eq!(
engine.idempotency_header("api.nokey.example").as_deref(),
Some("X-Idem")
);
engine.configure(&json!({})).expect("valid policy");
assert_eq!(engine.idempotency_header("api.stripe.com"), None);
}
struct Fixture {
_dir: TempDir,
clock: ManualClock,
journal: Arc<dyn Journal>,
manager: FlowManager,
desc: FlowDescriptor,
}
fn fixture() -> Fixture {
let dir = TempDir::new().expect("tempdir");
let clock = ManualClock::new(T0);
let journal: Arc<dyn Journal> =
Arc::new(SqliteJournal::open(dir.path().join("journal.db"), clock.clone()).expect("open"));
let engine = Engine::new();
engine
.configure(&json!({
"flows": { "on_nondeterminism": "fail" },
"target": { "api.pay.example": { "idempotency": { "header": "Idempotency-Key" } } }
}))
.expect("valid policy");
let clock_dyn: Arc<dyn Clock> = Arc::new(clock.clone());
let manager = FlowManager::new(
Arc::new(engine),
Arc::clone(&journal),
clock_dyn,
ProcessId::new("host-idem:pid-1"),
);
let desc = FlowDescriptor {
entrypoint: String::from("py:billing.run:main"),
args_hash: String::from("ah-idem"),
explicit_key: None,
code_hash: None,
};
Fixture {
_dir: dir,
clock,
journal,
manager,
desc,
}
}
fn payload_value(bytes: &[u8]) -> Value {
let envelope: Value = rmp_serde::from_slice(bytes).expect("payload decodes");
assert_eq!(envelope["schema"], "keel.step/v1", "schema-tagged payload");
envelope["payload"].clone()
}
async fn run1_completes_then_crashes_mid_step(fx: &Fixture) {
let mut handle = fx.manager.enter_flow(&fx.desc).expect("enter");
assert_eq!(
handle.recorded_idempotency_key("api.pay.example#c1"),
None,
"a fresh step has no recorded key"
);
let out = handle
.execute_step_with_idempotency_key(
&request("api.pay.example", "c1"),
Some("ik-1"),
|_attempt: u32| async {
AttemptResult::Ok {
payload: json!({ "charge": "ch_1" }),
}
},
)
.await;
assert_eq!(out.result, "ok");
{
let req = request("api.pay.example", "c2");
let fut = handle.execute_step_with_idempotency_key(&req, Some("ik-2"), |_attempt: u32| {
std::future::pending::<AttemptResult>()
});
let mut fut = pin!(fut);
let mut cx = Context::from_waker(Waker::noop());
assert!(
matches!(fut.as_mut().poll(&mut cx), Poll::Pending),
"the hung step must be in flight, not resolved"
);
}
drop(handle); }
#[tokio::test(start_paused = true)]
async fn crashed_running_step_journals_and_resurfaces_its_key() {
let fx = fixture();
run1_completes_then_crashes_mid_step(&fx).await;
fx.clock.advance(31_000);
let (key, running) = fx
.journal
.step_at(&fx.desc.flow_id(), 2)
.expect("read")
.expect("step 2 recorded");
assert_eq!(key.as_str(), "api.pay.example#c2");
assert_eq!(running.status, StepStatus::Running);
let payload = payload_value(running.payload.as_deref().expect("running payload"));
assert_eq!(payload, json!({ "idempotency_key": "ik-2" }));
let mut handle = fx.manager.enter_flow(&fx.desc).expect("resume");
let effect_calls = Arc::new(AtomicUsize::new(0));
let calls = Arc::clone(&effect_calls);
let out = handle
.execute_step_with_idempotency_key(
&request("api.pay.example", "c1"),
Some("ik-unused"),
move |_attempt: u32| {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
AttemptResult::Ok {
payload: json!({ "charge": "never" }),
}
}
},
)
.await;
assert_eq!(out.result, "ok");
assert_eq!(out.payload, Some(json!({ "charge": "ch_1" })));
assert_eq!(
effect_calls.load(Ordering::SeqCst),
0,
"a substituted step must not fire its effect"
);
assert_eq!(
handle.recorded_idempotency_key("api.pay.example#other"),
None
);
assert_eq!(
handle
.recorded_idempotency_key("api.pay.example#c2")
.as_deref(),
Some("ik-2")
);
let out = handle
.execute_step_with_idempotency_key(
&request("api.pay.example", "c2"),
Some("ik-2"), |_attempt: u32| async {
AttemptResult::Ok {
payload: json!({ "charge": "ch_2" }),
}
},
)
.await;
assert_eq!(out.result, "ok");
assert_eq!(out.attempts, 1);
handle.complete_success().unwrap();
let (_, terminal) = fx
.journal
.step_at(&fx.desc.flow_id(), 2)
.expect("read")
.expect("step 2 recorded");
assert_eq!(terminal.status, StepStatus::Ok);
assert_eq!(
payload_value(terminal.payload.as_deref().expect("terminal payload")),
json!({ "charge": "ch_2" })
);
}
#[tokio::test(start_paused = true)]
async fn steps_without_a_key_journal_no_payload_while_running() {
let fx = fixture();
let mut handle = fx.manager.enter_flow(&fx.desc).expect("enter");
let out = handle
.execute_step(
&request("api.pay.example", "plain"),
|_attempt: u32| async {
AttemptResult::Ok {
payload: json!({ "ok": true }),
}
},
)
.await;
assert_eq!(out.result, "ok");
handle.complete_success().unwrap();
let replay = fx.manager.enter_flow(&fx.desc).expect("pure replay");
assert!(replay.is_replay_only());
assert_eq!(
replay.recorded_idempotency_key("api.pay.example#plain"),
None
);
}