use chrono::{SubsecRound, Utc};
use skilj_core::bootstrap::ContextCreator;
use skilj_core::db::{self, Pool, SubmitCommandOutcome};
use skilj_core::event_cache::EventCache;
use skilj_core::event_store::{
BoundedContext, BoundedContextStatus, CommandType, Event, EventBroadcaster, EventOrigin,
EventType,
};
use skilj_core::plugin::{CommandDispatcher, ProjectionDispatcher};
use skilj_core::projections::Projection;
use skilj_core::shared::{
generate_token_id, CommandDecision, EventSpec, Metadata, Tag, TagMapping,
};
use std::sync::atomic::{AtomicUsize, Ordering};
struct TestCommandDispatcher {
calls: AtomicUsize,
}
impl TestCommandDispatcher {
fn new() -> Self {
Self {
calls: AtomicUsize::new(0),
}
}
fn call_count(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
impl CommandDispatcher for TestCommandDispatcher {
fn dispatch(
&self,
_bounded_context: &str,
command_type: &str,
payload: &str,
matching_events: &[Event],
) -> Option<skilj_core::error::Result<CommandDecision>> {
self.calls.fetch_add(1, Ordering::SeqCst);
match command_type {
"ShipOrder" => {
if matching_events.is_empty() {
Some(Ok(CommandDecision::Accepted {
events: vec![EventSpec {
event_type: "OrderShipped".to_string(),
payload: serde_json::from_str(payload)
.expect("test payload is always JSON"),
}],
}))
} else {
Some(Ok(CommandDecision::Rejected {
reason: "already shipped".to_string(),
kind: "already_shipped".to_string(),
}))
}
}
"TriggerUnregisteredEvent" => Some(Ok(CommandDecision::Accepted {
events: vec![EventSpec {
event_type: "NeverRegistered".to_string(),
payload: serde_json::json!({}),
}],
})),
"TriggerTwoEvents" => Some(Ok(CommandDecision::Accepted {
events: vec![
EventSpec {
event_type: "OrderShipped".to_string(),
payload: serde_json::json!({"order_id": "first"}),
},
EventSpec {
event_type: "PoisonEvent".to_string(),
payload: serde_json::json!({}),
},
],
})),
_ => None,
}
}
fn required_role(
&self,
_bounded_context: &str,
_command_type: &str,
) -> Option<Option<&'static str>> {
Some(None)
}
fn snapshot_name(
&self,
_bounded_context: &str,
_command_type: &str,
) -> Option<Option<&'static str>> {
Some(None)
}
fn dispatch_from_snapshot(
&self,
_bounded_context: &str,
_command_type: &str,
_payload: &str,
_snapshot_state_json: &str,
_events_since_snapshot: &[Event],
_context: &skilj_core::plugin::DecisionContext,
) -> Option<skilj_core::error::Result<CommandDecision>> {
unreachable!("no fixture in this file opts into snapshot()")
}
}
struct TestProjectionDispatcher;
impl ProjectionDispatcher for TestProjectionDispatcher {
fn keys(
&self,
_bounded_context: &str,
projection_name: &str,
_event: &Event,
) -> Option<Vec<String>> {
match projection_name {
"Poisonable" => Some(vec![String::new()]),
"ShipCount" => Some(if _event.event_type.name == "OrderShipped" {
vec![String::new()]
} else {
Vec::new()
}),
_ => None,
}
}
fn project(
&self,
_bounded_context: &str,
projection_name: &str,
state_json: &str,
event: &Event,
_key: &str,
) -> Option<skilj_core::error::Result<String>> {
match projection_name {
"Poisonable" if event.event_type.name == "PoisonEvent" => {
Some(Err(skilj_core::event_store::Error::UnregisteredEventType(
"deliberate test poison pill".to_string(),
)
.into()))
}
"Poisonable" => Some(Ok(state_json.to_string())),
"ShipCount" => Some(Ok((state_json.parse::<i64>().unwrap() + 1).to_string())),
_ => None,
}
}
fn default_state(&self, _bounded_context: &str, projection_name: &str) -> Option<String> {
match projection_name {
"Poisonable" | "ShipCount" => Some("0".to_string()),
_ => None,
}
}
fn owner_tag_key(
&self,
_bounded_context: &str,
projection_name: &str,
) -> Option<Option<&'static str>> {
match projection_name {
"Poisonable" => Some(None),
"ShipCount" => Some(Some("order")),
_ => None,
}
}
fn requires_capability(
&self,
_bounded_context: &str,
projection_name: &str,
) -> Option<Option<&'static str>> {
match projection_name {
"Poisonable" | "ShipCount" => Some(None),
_ => None,
}
}
}
struct TestDb {
pool: Pool,
}
static TEST_DB: tokio::sync::OnceCell<Option<TestDb>> = tokio::sync::OnceCell::const_new();
fn runtime() -> &'static tokio::runtime::Runtime {
static RUNTIME: std::sync::OnceLock<tokio::runtime::Runtime> = std::sync::OnceLock::new();
RUNTIME.get_or_init(|| {
tokio::runtime::Runtime::new()
.expect("failed to build a tokio runtime for submit_command tests")
})
}
async fn test_pool() -> Option<Pool> {
TEST_DB
.get_or_init(provision)
.await
.as_ref()
.map(|db| db.pool.clone())
}
async fn provision() -> Option<TestDb> {
let url = skilj_test_support::database_url("skilj_submit_command_test").await?;
let pool = match db::connect(&url).await {
Ok(pool) => pool,
Err(e) => {
eprintln!("skipping: connecting to the test database failed: {e}");
return None;
}
};
if let Err(e) = db::migrate(&pool).await {
eprintln!("skipping: migrating the test database failed: {e}");
return None;
}
Some(TestDb { pool })
}
fn unique_name(prefix: &str) -> String {
format!("{prefix}_{}", generate_token_id())
}
fn test_now() -> chrono::DateTime<Utc> {
Utc::now().trunc_subsecs(6)
}
async fn seed_bounded_context(pool: &Pool) -> BoundedContext {
let bc = BoundedContext {
name: unique_name("bc"),
status: BoundedContextStatus::Active,
created_at: test_now(),
created_by: ContextCreator::SystemCreator,
template: None,
};
db::insert_bounded_context(pool, &bc).await.unwrap();
bc
}
async fn seed_order_shipped_event_type(pool: &Pool, bc: &BoundedContext) -> EventType {
let et = EventType {
bounded_context: bc.clone(),
name: "OrderShipped".to_string(),
schema: r#"{"properties":{"order_id":{"type":"string"}}}"#.to_string(),
schema_version: 1,
tag_mappings: vec![TagMapping {
key: "order".to_string(),
field: "order_id".to_string(),
blind: false,
}],
owner_tag_key: None,
sensitive_fields: Vec::new(),
private_fields: Vec::new(),
external_creation_allowed: true,
direct_creation_allowed: true,
system_triggered_allowed: false,
system_triggered_schedule: None,
missed_occurrence_policy: None,
schedule_position: None,
last_fired_at: None,
event_read_allowed: true,
};
db::upsert_event_type(pool, &et).await.unwrap();
et
}
async fn seed_poison_event_type(pool: &Pool, bc: &BoundedContext) -> EventType {
let et = EventType {
bounded_context: bc.clone(),
name: "PoisonEvent".to_string(),
schema: r#"{"properties":{}}"#.to_string(),
schema_version: 1,
tag_mappings: Vec::new(),
owner_tag_key: None,
sensitive_fields: Vec::new(),
private_fields: Vec::new(),
external_creation_allowed: true,
direct_creation_allowed: true,
system_triggered_allowed: false,
system_triggered_schedule: None,
missed_occurrence_policy: None,
schedule_position: None,
last_fired_at: None,
event_read_allowed: true,
};
db::upsert_event_type(pool, &et).await.unwrap();
et
}
async fn seed_command_type(pool: &Pool, bc: &BoundedContext, name: &str) -> CommandType {
let ct = CommandType {
bounded_context: bc.clone(),
name: name.to_string(),
schema: r#"{"properties":{"order_id":{"type":"string"}}}"#.to_string(),
schema_version: 1,
tag_mappings: vec![TagMapping {
key: "order".to_string(),
field: "order_id".to_string(),
blind: false,
}],
owner_tag_key: None,
sensitive_fields: Vec::new(),
private_fields: Vec::new(),
rest_trigger_allowed: true,
};
db::upsert_command_type(pool, &ct).await.unwrap();
ct
}
async fn insert_concurrent_event(
pool: &Pool,
bc: &BoundedContext,
et: &EventType,
tag_value: &str,
) {
let seq = db::next_sequence(pool, &bc.name).await.unwrap();
let e = Event {
bounded_context: bc.clone(),
event_type: et.clone(),
payload: format!(r#"{{"order_id":"{tag_value}"}}"#),
metadata: Metadata {
r#type: et.name.clone(),
version: et.schema_version,
client_id: "concurrent-writer".to_string(),
created_at: test_now(),
correlation_id: None,
causation_id: None,
},
sequence: seq,
tags: vec![Tag {
key: "order".to_string(),
value: Some(tag_value.to_string()),
}],
encryption_keys: Vec::new(),
origin: EventOrigin::DirectlyCreated,
};
db::insert_event(pool, &e, None).await.unwrap();
}
#[test]
fn submit_command_redispatches_and_rejects_on_a_genuine_dcb_conflict() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let consistency_tags =
skilj_core::event_store::derive_plain_tags(&ct.tag_mappings, payload);
let bounded_context_events: Vec<Event> = Vec::new();
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
assert!(matches!(initial_decision, CommandDecision::Accepted { .. }));
assert_eq!(dispatcher.call_count(), 1);
insert_concurrent_event(&pool, &bc, &et, "A").await;
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&bounded_context_events,
&consistency_tags,
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert_eq!(dispatcher.call_count(), 2);
match outcome {
SubmitCommandOutcome::Rejected {
kind,
matching_events,
..
} => {
assert_eq!(kind, "already_shipped");
assert_eq!(matching_events.len(), 1);
assert_eq!(matching_events[0].tags[0].value.as_deref(), Some("A"));
}
SubmitCommandOutcome::Accepted { .. } => {
panic!("a genuine DCB conflict was not caught - stale decision was used")
}
SubmitCommandOutcome::Deduplicated { .. } => {
panic!("no idempotency_key was given - Deduplicated must be unreachable")
}
other => panic!("unexpected {other:?}"),
}
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].sequence, 0);
});
}
#[test]
fn submit_command_does_not_redispatch_for_an_unrelated_concurrent_event() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let consistency_tags =
skilj_core::event_store::derive_plain_tags(&ct.tag_mappings, payload);
let bounded_context_events: Vec<Event> = Vec::new();
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
insert_concurrent_event(&pool, &bc, &et, "B").await;
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&bounded_context_events,
&consistency_tags,
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert_eq!(
dispatcher.call_count(),
1,
"an unrelated concurrent event must not cost a redispatch"
);
assert!(matches!(outcome, SubmitCommandOutcome::Accepted { .. }));
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 2);
assert_eq!(events[1].sequence, 1);
});
}
#[test]
fn submit_command_persists_a_real_command_id_that_round_trips() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted { command, events } = outcome else {
panic!("this submission has no reason to be rejected");
};
assert!(!command.id.is_empty());
for event in &events {
match &event.origin {
EventOrigin::CommandTriggered { command: origin } => {
assert_eq!(origin.id, command.id);
}
_ => panic!("every event submit_command produces here is command-triggered"),
}
}
let reloaded = db::list_commands_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(reloaded.len(), 1);
assert_eq!(reloaded[0].id, command.id);
});
}
#[test]
fn submit_command_with_a_repeated_idempotency_key_short_circuits_to_the_original_outcome() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let key = "retry-key-1";
let first_decision = dispatcher.dispatch(&bc.name, &ct.name, payload, &[]).unwrap().unwrap();
let first_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
first_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted {
events: first_events,
..
} = first_outcome
else {
panic!("this submission has no reason to be rejected");
};
let first_sequences: Vec<i64> = first_events.iter().map(|e| e.sequence).collect();
let second_decision = dispatcher.dispatch(&bc.name, &ct.name, payload, &[]).unwrap().unwrap();
let second_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
second_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Deduplicated {
triggered_event_sequences,
} = second_outcome
else {
panic!("a repeated idempotency key must short-circuit to Deduplicated, got {second_outcome:?}");
};
assert_eq!(triggered_event_sequences, first_sequences);
let commands = db::list_commands_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(
commands.len(),
1,
"a dedup hit must not insert a second Command row"
);
});
}
#[test]
fn submit_command_with_the_same_idempotency_key_from_two_different_clients_does_not_collide() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let key = "invoice-2024-01";
let tenant_a_decision = dispatcher
.dispatch(&bc.name, &ct.name, r#"{"order_id":"A"}"#, &[])
.unwrap()
.unwrap();
let tenant_a_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
r#"{"order_id":"A"}"#,
"tenant-a",
None,
None,
&[],
&[],
&[],
tenant_a_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert!(
matches!(tenant_a_outcome, SubmitCommandOutcome::Accepted { .. }),
"tenant a's first use of this key must be accepted, got {tenant_a_outcome:?}"
);
let tenant_b_decision = dispatcher
.dispatch(&bc.name, &ct.name, r#"{"order_id":"B"}"#, &[])
.unwrap()
.unwrap();
let tenant_b_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
r#"{"order_id":"B"}"#,
"tenant-b",
None,
None,
&[],
&[],
&[],
tenant_b_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted {
events: tenant_b_events,
..
} = tenant_b_outcome
else {
panic!(
"tenant b's own, unrelated submission must be decided for real, not \
deduplicated against tenant a's - got {tenant_b_outcome:?}"
);
};
assert_eq!(
tenant_b_events.len(),
1,
"tenant b's own OrderShipped must actually have been inserted"
);
let commands = db::list_commands_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(
commands.len(),
2,
"two different clients' own submissions under the same key string \
must both actually persist"
);
let tenant_a_retry_decision = dispatcher
.dispatch(&bc.name, &ct.name, r#"{"order_id":"A"}"#, &[])
.unwrap()
.unwrap();
let tenant_a_retry_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
r#"{"order_id":"A"}"#,
"tenant-a",
None,
None,
&[],
&[],
&[],
tenant_a_retry_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted {
events: tenant_a_events,
..
} = tenant_a_outcome
else {
unreachable!("checked above");
};
let tenant_a_sequences: Vec<i64> = tenant_a_events.iter().map(|e| e.sequence).collect();
let SubmitCommandOutcome::Deduplicated {
triggered_event_sequences,
} = tenant_a_retry_outcome
else {
panic!(
"tenant a retrying their own key must still deduplicate against \
their own answer, got {tenant_a_retry_outcome:?}"
);
};
assert_eq!(
triggered_event_sequences, tenant_a_sequences,
"must dedup to tenant a's own sequences, never tenant b's"
);
});
}
#[test]
fn migrate_idempotency_keys_client_id_scoping_retires_pre_migration_rows() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let schema = format!("\"bc_{}\"", bc.name);
sqlx::query(sqlx::AssertSqlSafe(format!(
"DROP TABLE {schema}.idempotency_keys"
)))
.execute(&pool)
.await
.unwrap();
sqlx::query(sqlx::AssertSqlSafe(format!(
"CREATE TABLE {schema}.idempotency_keys (
command_type_name TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
triggered_event_sequences BIGINT[] NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (command_type_name, idempotency_key)
)"
)))
.execute(&pool)
.await
.unwrap();
sqlx::query(sqlx::AssertSqlSafe(format!(
"INSERT INTO {schema}.idempotency_keys \
(command_type_name, idempotency_key, triggered_event_sequences, created_at) \
VALUES ($1, $2, $3, $4)"
)))
.bind(&ct.name)
.bind("legacy-key")
.bind([42_i64].as_slice())
.bind(test_now())
.execute(&pool)
.await
.unwrap();
db::ensure_idempotency_keys_table(&pool, &bc.name)
.await
.unwrap();
db::migrate_idempotency_keys_client_id_scoping(&pool, &bc.name)
.await
.unwrap();
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"whoever-originally-submitted-this",
None,
None,
&[],
&[],
&[],
decision,
None,
test_now(),
None,
&[],
Some("legacy-key"),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted {
events: retried_events,
..
} = outcome
else {
panic!(
"a pre-migration row must never be matched again - decide() runs for \
real, got {outcome:?}"
);
};
assert_eq!(
retried_events.len(),
1,
"the retry actually persisted a real new event, not a cached answer"
);
let legacy_row_still_present: (i64,) = sqlx::query_as(sqlx::AssertSqlSafe(format!(
"SELECT triggered_event_sequences[1] FROM {schema}.idempotency_keys \
WHERE client_id = '' AND idempotency_key = 'legacy-key'"
)))
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(legacy_row_still_present.0, 42);
let fresh_decision = dispatcher
.dispatch(&bc.name, &ct.name, r#"{"order_id":"B"}"#, &[])
.unwrap()
.unwrap();
let fresh_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
r#"{"order_id":"B"}"#,
"a-different-client",
None,
None,
&[],
&[],
&[],
fresh_decision,
None,
test_now(),
None,
&[],
Some("a-different-key-this-client-owns"),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert!(
matches!(fresh_outcome, SubmitCommandOutcome::Accepted { .. }),
"a different client's own, never-before-seen key must be decided for real, \
got {fresh_outcome:?}"
);
});
}
#[test]
fn migrate_idempotency_keys_client_id_scoping_is_safe_under_concurrent_callers() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let schema = format!("\"bc_{}\"", bc.name);
sqlx::query(sqlx::AssertSqlSafe(format!(
"DROP TABLE {schema}.idempotency_keys"
)))
.execute(&pool)
.await
.unwrap();
sqlx::query(sqlx::AssertSqlSafe(format!(
"CREATE TABLE {schema}.idempotency_keys (
command_type_name TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
triggered_event_sequences BIGINT[] NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (command_type_name, idempotency_key)
)"
)))
.execute(&pool)
.await
.unwrap();
let handles: Vec<_> = (0..8)
.map(|_| {
let pool = pool.clone();
let name = bc.name.clone();
tokio::spawn(async move {
db::migrate_idempotency_keys_client_id_scoping(&pool, &name).await
})
})
.collect();
for handle in handles {
handle
.await
.expect("task panicked")
.expect("the advisory lock must serialise concurrent migrators, not error");
}
let pk_columns: Vec<(String,)> = sqlx::query_as(
"SELECT column_name FROM information_schema.key_column_usage \
WHERE table_schema = $1 AND table_name = 'idempotency_keys' \
AND constraint_name = 'idempotency_keys_pkey' \
ORDER BY ordinal_position",
)
.bind(format!("bc_{}", bc.name))
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(
pk_columns,
vec![
("command_type_name".to_string(),),
("client_id".to_string(),),
("idempotency_key".to_string(),),
],
"the primary key must end up with exactly these three columns, in this \
order, regardless of how many concurrent callers raced to get there"
);
});
}
#[test]
fn submit_command_without_an_idempotency_key_still_double_processes_a_repeated_submission() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
for _ in 0..2 {
let decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert!(
matches!(outcome, SubmitCommandOutcome::Accepted { .. }),
"no key given - every submission is new, exactly as before this feature existed"
);
}
let commands = db::list_commands_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(
commands.len(),
2,
"two keyless submissions must both persist"
);
});
}
#[test]
fn ensure_idempotency_keys_table_patches_a_bounded_context_provisioned_before_this_feature() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let schema = format!("\"bc_{}\"", bc.name);
sqlx::query(sqlx::AssertSqlSafe(format!(
"DROP TABLE {schema}.idempotency_keys"
)))
.execute(&pool)
.await
.unwrap();
db::ensure_idempotency_keys_table(&pool, &bc.name)
.await
.unwrap();
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = r#"{"order_id":"A"}"#;
let key = "post-patch-retry";
let first_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
first_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let second_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let second_outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
second_decision,
None,
test_now(),
None,
&[],
Some(key),
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert!(
matches!(second_outcome, SubmitCommandOutcome::Deduplicated { .. }),
"idempotency must work normally after the patch, not just leave the table present"
);
});
}
#[test]
fn submit_command_leaves_no_sequence_gap_when_process_command_fails() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let ct = seed_command_type(&pool, &bc, "TriggerUnregisteredEvent").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = "{}";
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let result = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await;
assert!(result.is_err());
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 0);
});
}
#[test]
fn submit_command_rolls_back_the_command_and_every_event_together_when_a_later_event_fails_to_insert(
) {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let order_shipped = seed_order_shipped_event_type(&pool, &bc).await;
let poison = seed_poison_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "TriggerTwoEvents").await;
let projection = Projection {
bounded_context: bc.clone(),
name: "Poisonable".to_string(),
schema: r#"{"properties":{}}"#.to_string(),
schema_version: 1,
consumed_event_types: vec![order_shipped, poison],
sync: true,
caught_up_to: None,
requires_capability: None,
};
db::upsert_projection(&pool, &projection).await.unwrap();
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = "{}";
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let result = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await;
assert!(
result.is_err(),
"the poison pill's Some(Err(..)) must fail the whole submission"
);
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert!(events.is_empty());
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 0);
});
}
#[test]
fn submit_command_batches_sequence_allocation_for_a_multi_event_command() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
seed_poison_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "TriggerTwoEvents").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
let payload = "{}";
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&[],
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
None,
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
let SubmitCommandOutcome::Accepted { events, .. } = outcome else {
panic!("no Poisonable projection is registered, so nothing rejects this");
};
assert_eq!(events.len(), 2);
assert_eq!(events[0].event_type.name, "OrderShipped");
assert_eq!(events[0].sequence, 0);
assert_eq!(events[1].event_type.name, "PoisonEvent");
assert_eq!(events[1].sequence, 1);
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 2);
});
}
fn batched(
command_type: &CommandType,
payload: &str,
initial_decision: CommandDecision,
consistency_tags: Vec<Tag>,
) -> db::BatchedCommand {
db::BatchedCommand {
command_type: command_type.clone(),
payload: payload.to_string(),
client_id: "client-1".to_string(),
correlation_id: None,
causation_id: None,
bounded_context_events: Vec::new(),
covered_through: None,
consistency_tags,
matching_events: Vec::new(),
initial_decision,
now: test_now(),
snapshot: None,
snapshot_states: Vec::new(),
idempotency_key: None,
event_types_by_name: std::collections::HashMap::new(),
resolved: std::collections::HashMap::new(),
}
}
#[test]
fn submit_command_batch_detects_a_dcb_conflict_between_two_commands_in_the_same_batch() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let payload = r#"{"order_id":"A"}"#;
let tags = vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}];
let decision_1 = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let decision_2 = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
assert_eq!(dispatcher.call_count(), 2);
let batch = vec![
batched(&ct, payload, decision_1, tags.clone()),
batched(&ct, payload, decision_2, tags),
];
let mut results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
batch,
)
.await
.unwrap();
assert_eq!(results.len(), 2);
let second = results.pop().unwrap().unwrap();
let first = results.pop().unwrap().unwrap();
match first {
SubmitCommandOutcome::Accepted { events, .. } => {
assert_eq!(events.len(), 1);
assert_eq!(events[0].sequence, 0);
}
other => panic!("expected the first command to ship the order, got {other:?}"),
}
match second {
SubmitCommandOutcome::Rejected { reason, .. } => {
assert_eq!(reason, "already shipped");
}
other => panic!(
"expected the second command to be rejected as a same-batch DCB conflict, got \
{other:?}"
),
}
assert_eq!(dispatcher.call_count(), 3);
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 1);
});
}
#[test]
fn submit_command_batch_isolates_a_failing_commands_savepoint_from_its_batch_mates() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
seed_poison_event_type(&pool, &bc).await;
let ship_order = seed_command_type(&pool, &bc, "ShipOrder").await;
let trigger_two = seed_command_type(&pool, &bc, "TriggerTwoEvents").await;
let projection = Projection {
bounded_context: bc.clone(),
name: "Poisonable".to_string(),
schema: r#"{"properties":{}}"#.to_string(),
schema_version: 1,
consumed_event_types: Vec::new(),
sync: true,
caught_up_to: None,
requires_capability: None,
};
db::upsert_projection(&pool, &projection).await.unwrap();
let dispatcher = TestCommandDispatcher::new();
let ship_payload = r#"{"order_id":"A"}"#;
let ship_decision = dispatcher
.dispatch(&bc.name, &ship_order.name, ship_payload, &[])
.unwrap()
.unwrap();
let poison_decision = dispatcher
.dispatch(&bc.name, &trigger_two.name, "{}", &[])
.unwrap()
.unwrap();
let batch = vec![
batched(
&ship_order,
ship_payload,
ship_decision,
vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}],
),
batched(&trigger_two, "{}", poison_decision, Vec::new()),
];
let mut results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
batch,
)
.await
.unwrap();
assert_eq!(results.len(), 2);
let second = results.pop().unwrap();
let first = results.pop().unwrap();
match first.unwrap() {
SubmitCommandOutcome::Accepted { events, .. } => {
assert_eq!(events.len(), 1);
assert_eq!(events[0].sequence, 0);
}
other => panic!("expected the first command to succeed, got {other:?}"),
}
assert!(
second.is_err(),
"the poison pill's Some(Err(..)) must fail only its own command"
);
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].event_type.name, "OrderShipped");
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 1);
});
}
#[test]
fn submit_command_batch_recycles_a_failed_commands_sequence_numbers_for_a_later_command_in_the_same_batch(
) {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
seed_poison_event_type(&pool, &bc).await;
let ship_order = seed_command_type(&pool, &bc, "ShipOrder").await;
let trigger_two = seed_command_type(&pool, &bc, "TriggerTwoEvents").await;
let projection = Projection {
bounded_context: bc.clone(),
name: "Poisonable".to_string(),
schema: r#"{"properties":{}}"#.to_string(),
schema_version: 1,
consumed_event_types: Vec::new(),
sync: true,
caught_up_to: None,
requires_capability: None,
};
db::upsert_projection(&pool, &projection).await.unwrap();
let dispatcher = TestCommandDispatcher::new();
let ship_a_payload = r#"{"order_id":"A"}"#;
let ship_a_decision = dispatcher
.dispatch(&bc.name, &ship_order.name, ship_a_payload, &[])
.unwrap()
.unwrap();
let poison_decision = dispatcher
.dispatch(&bc.name, &trigger_two.name, "{}", &[])
.unwrap()
.unwrap();
let ship_b_payload = r#"{"order_id":"B"}"#;
let ship_b_decision = dispatcher
.dispatch(&bc.name, &ship_order.name, ship_b_payload, &[])
.unwrap()
.unwrap();
let batch = vec![
batched(
&ship_order,
ship_a_payload,
ship_a_decision,
vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}],
),
batched(&trigger_two, "{}", poison_decision, Vec::new()),
batched(
&ship_order,
ship_b_payload,
ship_b_decision,
vec![Tag {
key: "order".to_string(),
value: Some("B".to_string()),
}],
),
];
let mut results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
batch,
)
.await
.unwrap();
assert_eq!(results.len(), 3);
let third = results.pop().unwrap();
let second = results.pop().unwrap();
let first = results.pop().unwrap();
match first.unwrap() {
SubmitCommandOutcome::Accepted { events, .. } => {
assert_eq!(events.len(), 1);
assert_eq!(events[0].sequence, 0);
}
other => panic!("expected the first command to succeed, got {other:?}"),
}
assert!(
second.is_err(),
"the poison pill's Some(Err(..)) must fail only its own command"
);
match third.unwrap() {
SubmitCommandOutcome::Accepted { events, .. } => {
assert_eq!(events.len(), 1);
assert_eq!(
events[0].sequence, 1,
"the third command's own event must land on the sequence number the \
failed second command drew and gave back, not a fresh one past it - \
otherwise sequence 1 would be a permanent gap"
);
}
other => panic!("expected the third command to succeed, got {other:?}"),
}
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 2);
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 2);
});
}
#[test]
fn submit_command_batch_deduplicates_a_repeated_idempotency_key_shared_by_two_commands_in_the_same_batch(
) {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let payload = r#"{"order_id":"A"}"#;
let decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
let mut first = batched(
&ct,
payload,
decision.clone(),
vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}],
);
first.idempotency_key = Some("retry-key".to_string());
let mut second = batched(
&ct,
payload,
decision,
vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}],
);
second.idempotency_key = Some("retry-key".to_string());
let mut results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
vec![first, second],
)
.await
.unwrap();
assert_eq!(results.len(), 2);
let second = results.pop().unwrap().unwrap();
let first = results.pop().unwrap().unwrap();
let first_sequence = match first {
SubmitCommandOutcome::Accepted { events, .. } => {
assert_eq!(events.len(), 1);
events[0].sequence
}
other => panic!("expected the first command to ship the order, got {other:?}"),
};
match second {
SubmitCommandOutcome::Deduplicated {
triggered_event_sequences,
} => {
assert_eq!(
triggered_event_sequences,
vec![first_sequence],
"the second command's own idempotency lookup must see the first \
command's already-inserted (if not yet durably committed) key"
);
}
other => panic!("expected the second, same-key command to deduplicate, got {other:?}"),
}
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
assert_eq!(events.len(), 1);
});
}
#[test]
fn a_known_read_position_still_catches_a_conflict_committed_after_it() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
insert_concurrent_event(&pool, &bc, &et, "X").await;
insert_concurrent_event(&pool, &bc, &et, "Y").await;
let payload = r#"{"order_id":"A"}"#;
let consistency_tags = skilj_core::event_store::derive_plain_tags(&ct.tag_mappings, payload);
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap();
insert_concurrent_event(&pool, &bc, &et, "A").await;
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&[],
&consistency_tags,
&[],
initial_decision,
None,
test_now(),
None,
&[],
None,
Some(1),
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert_eq!(dispatcher.call_count(), 2, "redispatched");
assert!(
matches!(outcome, SubmitCommandOutcome::Rejected { ref matching_events, .. } if matching_events.len() == 1),
"the conflict after the read's position was missed: {outcome:?}"
);
});
}
#[test]
fn an_event_the_read_already_holds_above_its_position_is_not_counted_twice() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let broadcaster = EventBroadcaster::new(16);
let event_cache = EventCache::new(1000);
insert_concurrent_event(&pool, &bc, &et, "A").await;
let held = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
let payload = r#"{"order_id":"A"}"#;
let consistency_tags = skilj_core::event_store::derive_plain_tags(&ct.tag_mappings, payload);
let initial_decision = dispatcher
.dispatch(&bc.name, &ct.name, payload, &held)
.unwrap()
.unwrap();
let outcome = db::submit_command(
&pool,
&dispatcher,
&TestProjectionDispatcher,
&broadcaster,
&event_cache,
&ct,
payload,
"client-1",
None,
None,
&held,
&consistency_tags,
&held,
initial_decision,
None,
test_now(),
None,
&[],
None,
Some(-1),
&skilj_core::plugin::DecisionContext::at(chrono::Utc::now()),
)
.await
.unwrap();
assert_eq!(dispatcher.call_count(), 1, "nothing new - no redispatch");
assert!(
matches!(outcome, SubmitCommandOutcome::Rejected { ref matching_events, .. } if matching_events.len() == 1),
"the held event must count once: {outcome:?}"
);
});
}
#[test]
fn a_batch_written_as_a_set_links_its_events_and_records_its_keys() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let command = |order: &str, key: Option<&str>| {
let payload = format!(r#"{{"order_id":"{order}"}}"#);
let decision = dispatcher
.dispatch(&bc.name, &ct.name, &payload, &[])
.unwrap()
.unwrap();
let mut item = batched(
&ct,
&payload,
decision,
vec![Tag {
key: "order".to_string(),
value: Some(order.to_string()),
}],
);
item.idempotency_key = key.map(str::to_string);
item
};
let results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
vec![
command("B", Some("key-b")),
command("C", None),
command("D", None),
],
)
.await
.unwrap();
let mut accepted = Vec::new();
for result in results {
match result.unwrap() {
SubmitCommandOutcome::Accepted { command, events } => {
assert_eq!(events.len(), 1);
accepted.push((command.id.clone(), events[0].sequence));
}
other => panic!("expected every command to be accepted, got {other:?}"),
}
}
assert_eq!(
accepted.iter().map(|(_, s)| *s).collect::<Vec<_>>(),
[0, 1, 2]
);
assert_eq!(db::next_sequence(&pool, &bc.name).await.unwrap(), 3);
let events = db::list_events_for_bounded_context(&pool, &bc.name)
.await
.unwrap();
for (event, (command_id, sequence)) in events.iter().zip(&accepted) {
assert_eq!(event.sequence, *sequence);
match &event.origin {
EventOrigin::CommandTriggered { command } => {
assert_eq!(&command.id, command_id, "event {sequence}")
}
other => panic!("expected a command-triggered event, got {other:?}"),
}
}
let results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
vec![command("B", Some("key-b"))],
)
.await
.unwrap();
match results.into_iter().next().unwrap().unwrap() {
SubmitCommandOutcome::Deduplicated {
triggered_event_sequences,
} => assert_eq!(triggered_event_sequences, vec![0]),
other => panic!("expected the repeated key to deduplicate, got {other:?}"),
}
});
}
#[test]
fn one_recheck_per_batch_gives_each_command_only_its_own_conflicts() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let command = |order: &str| {
let payload = format!(r#"{{"order_id":"{order}"}}"#);
let decision = dispatcher
.dispatch(&bc.name, &ct.name, &payload, &[])
.unwrap()
.unwrap();
batched(
&ct,
&payload,
decision,
vec![Tag {
key: "order".to_string(),
value: Some(order.to_string()),
}],
)
};
let batch = vec![command("A"), command("B"), command("C")];
insert_concurrent_event(&pool, &bc, &et, "A").await;
insert_concurrent_event(&pool, &bc, &et, "C").await;
let results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
batch,
)
.await
.unwrap();
let outcomes: Vec<_> = results.into_iter().map(Result::unwrap).collect();
for (outcome, order) in outcomes.iter().zip(["A", "B", "C"]) {
match (order, outcome) {
("B", SubmitCommandOutcome::Accepted { events, .. }) => {
assert_eq!(events[0].sequence, 2)
}
(
_,
SubmitCommandOutcome::Rejected {
matching_events, ..
},
) if order != "B" => {
assert_eq!(matching_events.len(), 1, "{order}");
assert_eq!(matching_events[0].tags[0].value.as_deref(), Some(order));
}
(order, other) => panic!("unexpected outcome for {order}: {other:?}"),
}
}
});
}
#[test]
fn a_batch_folds_its_events_into_a_sync_projection_at_once() {
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
let et = seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
db::upsert_projection(
&pool,
&Projection {
bounded_context: bc.clone(),
name: "ShipCount".to_string(),
schema: r#"{"properties":{}}"#.to_string(),
schema_version: 1,
consumed_event_types: vec![et.clone()],
sync: true,
caught_up_to: None,
requires_capability: None,
},
)
.await
.unwrap();
let dispatcher = TestCommandDispatcher::new();
let command = |order: &str| {
let payload = format!(r#"{{"order_id":"{order}"}}"#);
let decision = dispatcher
.dispatch(&bc.name, &ct.name, &payload, &[])
.unwrap()
.unwrap();
batched(
&ct,
&payload,
decision,
vec![Tag {
key: "order".to_string(),
value: Some(order.to_string()),
}],
)
};
let results = db::submit_command_batch(
&pool,
&dispatcher,
&TestProjectionDispatcher,
None,
&bc.name,
vec![command("B"), command("C"), command("D")],
)
.await
.unwrap();
assert!(results
.iter()
.all(|r| matches!(r, Ok(SubmitCommandOutcome::Accepted { .. }))));
let (state, owner, as_of): (String, Option<String>, i64) =
sqlx::query_as(sqlx::AssertSqlSafe(format!(
"SELECT state, owner, as_of_sequence FROM \"bc_{}\".projection_state \
WHERE projection_name = 'ShipCount' AND key = ''",
bc.name
)))
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
(state.as_str(), owner.as_deref(), as_of),
("3", Some("D"), 2)
);
let projection = db::get_projection(&pool, &bc.name, "ShipCount")
.await
.unwrap()
.unwrap();
assert_eq!(projection.caught_up_to, Some(2));
});
}
#[derive(Clone, Default)]
struct Warnings(std::sync::Arc<std::sync::Mutex<Vec<String>>>);
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Warnings {
fn on_event(
&self,
event: &tracing::Event<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
) {
struct Fields(String);
impl tracing::field::Visit for Fields {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.0.push_str(&format!("{}={value:?} ", field.name()));
}
}
if *event.metadata().level() == tracing::Level::WARN
&& event.metadata().target().starts_with("skilj_core")
{
let mut fields = Fields(String::new());
event.record(&mut fields);
self.0.lock().unwrap().push(fields.0);
}
}
}
#[test]
fn a_slow_decide_again_under_the_lock_warns() {
use tracing_subscriber::layer::SubscriberExt;
let warnings = Warnings::default();
let _guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(warnings.clone()));
runtime().block_on(async {
let Some(pool) = test_pool().await else {
return;
};
let bc = seed_bounded_context(&pool).await;
seed_order_shipped_event_type(&pool, &bc).await;
let ct = seed_command_type(&pool, &bc, "ShipOrder").await;
let dispatcher = TestCommandDispatcher::new();
let payload = r#"{"order_id":"A"}"#;
let tags = vec![Tag {
key: "order".to_string(),
value: Some("A".to_string()),
}];
let decide = || {
dispatcher
.dispatch(&bc.name, &ct.name, payload, &[])
.unwrap()
.unwrap()
};
let batch = vec![
batched(&ct, payload, decide(), tags.clone()),
batched(&ct, payload, decide(), tags),
];
let slow = skilj_core::slow_command::SlowCommandWarnings::default()
.with_read(None)
.with_decide(Some(std::time::Duration::ZERO));
let leader_tx = db::begin_command_batch_leader_tx(&pool, &bc.name, None)
.await
.unwrap();
let results = db::commit_command_batch(
leader_tx,
&dispatcher,
&TestProjectionDispatcher,
None,
batch,
Some(&slow),
)
.await
.unwrap();
assert!(matches!(
results[1],
Ok(SubmitCommandOutcome::Rejected { .. })
));
let logged = warnings.0.lock().unwrap().clone();
assert_eq!(logged.len(), 1, "{logged:?}");
assert!(logged[0].contains("under_lock=true"), "{}", logged[0]);
assert!(logged[0].contains("events_decided_on=1"), "{}", logged[0]);
assert!(
logged[0].contains("command_type=ShipOrder"),
"{}",
logged[0]
);
});
}