use std::collections::{BTreeMap, HashSet};
use futures_util::future::join_all;
use super::{
Projection, ProjectionError, ProjectionHandle, ProjectionId, ProjectionResult,
ProjectionRuntime, ProjectionStatus, runtime,
};
struct InstanceBlueprint<Envelope> {
projection_id: ProjectionId,
projection: Projection<Envelope>,
}
pub struct ProjectionGroup<Envelope> {
projection_name: String,
instances: Vec<InstanceBlueprint<Envelope>>,
}
impl<Envelope> ProjectionGroup<Envelope>
where
Envelope: Send + Sync + 'static,
{
#[must_use]
pub fn for_keys<I, K, F>(projection_name: impl Into<String>, keys: I, template: F) -> Self
where
I: IntoIterator<Item = K>,
K: Into<String>,
F: Fn(ProjectionId) -> Projection<Envelope>,
{
let projection_name = projection_name.into();
let instances = keys
.into_iter()
.map(Into::into)
.map(|key| {
let projection_id = ProjectionId::new(projection_name.clone(), key);
let projection = template(projection_id.clone());
InstanceBlueprint {
projection_id,
projection,
}
})
.collect();
Self::from_instance_blueprints(projection_name, instances)
}
fn from_instance_blueprints(
projection_name: String,
instances: Vec<InstanceBlueprint<Envelope>>,
) -> Self {
Self {
projection_name,
instances,
}
}
#[must_use]
pub fn projection_name(&self) -> &str {
&self.projection_name
}
#[must_use]
pub fn len(&self) -> usize {
self.instances.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.instances.is_empty()
}
pub fn projection_ids(&self) -> impl ExactSizeIterator<Item = &ProjectionId> {
self.instances
.iter()
.map(|instance| &instance.projection_id)
}
pub async fn spawn(
self,
runtime_resources: ProjectionRuntime,
) -> ProjectionResult<ProjectionGroupHandle> {
self.validate(&runtime_resources)?;
let projection_ids = self
.instances
.iter()
.map(|instance| instance.projection_id.clone())
.collect::<Vec<_>>();
let registry_leases = runtime::reserve_projections(&projection_ids)?;
let mut handles = Vec::with_capacity(self.instances.len());
let materializations = self.instances.into_iter().zip(registry_leases);
for (instance, registry_lease) in materializations {
match runtime::spawn_reserved(
instance.projection,
runtime_resources.clone(),
registry_lease,
)
.await
{
Ok(handle) => handles.push(handle),
Err(error) => {
let _ = join_all(handles.iter().map(ProjectionHandle::stop_and_wait)).await;
return Err(error);
}
}
}
let instances = handles
.into_iter()
.map(|handle| (handle.projection_id().key().to_owned(), handle))
.collect();
Ok(ProjectionGroupHandle {
projection_name: self.projection_name,
instances,
})
}
pub async fn run_on(
self,
runtime_resources: ProjectionRuntime,
) -> ProjectionResult<ProjectionGroupHandle> {
self.spawn(runtime_resources).await
}
fn validate(&self, runtime_resources: &ProjectionRuntime) -> ProjectionResult<()> {
if self.instances.is_empty() {
return Err(ProjectionError::invalid_config(
"projection_group.keys",
"at least one projection key is required",
));
}
let mut projection_ids = HashSet::with_capacity(self.instances.len());
for instance in &self.instances {
instance.projection_id.validate()?;
if !projection_ids.insert(instance.projection_id.clone()) {
return Err(ProjectionError::invalid_config(
"projection_group.keys",
format!(
"duplicate projection key {:?}",
instance.projection_id.key()
),
));
}
if instance.projection.projection_id() != &instance.projection_id {
return Err(ProjectionError::invalid_config(
"projection_group.template.projection_id",
format!(
"template returned {}, expected {}",
instance.projection.projection_id(),
instance.projection_id
),
));
}
runtime::validate_spawn(&instance.projection, runtime_resources)?;
}
Ok(())
}
}
#[must_use = "dropping a group handle only requests best-effort instance stops"]
pub struct ProjectionGroupHandle {
projection_name: String,
instances: BTreeMap<String, ProjectionHandle>,
}
impl ProjectionGroupHandle {
#[must_use]
pub fn projection_name(&self) -> &str {
&self.projection_name
}
#[must_use]
pub fn len(&self) -> usize {
self.instances.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.instances.is_empty()
}
#[must_use]
pub fn instance(&self, projection_id: &ProjectionId) -> Option<&ProjectionHandle> {
(projection_id.name() == self.projection_name)
.then(|| self.instances.get(projection_id.key()))
.flatten()
.filter(|handle| handle.projection_id() == projection_id)
}
#[must_use]
pub fn instance_by_key(&self, key: &str) -> Option<&ProjectionHandle> {
self.instances.get(key)
}
pub fn instances(&self) -> impl ExactSizeIterator<Item = &ProjectionHandle> {
self.instances.values()
}
#[must_use]
pub fn statuses(&self) -> BTreeMap<ProjectionId, ProjectionStatus> {
self.instances
.values()
.map(|handle| (handle.projection_id().clone(), handle.status()))
.collect()
}
pub async fn pause_all(&self) -> ProjectionResult<()> {
first_error(join_all(self.instances.values().map(ProjectionHandle::pause)).await)
}
pub async fn resume_all(&self) -> ProjectionResult<()> {
first_error(join_all(self.instances.values().map(ProjectionHandle::resume)).await)
}
pub async fn stop_and_wait(&self) -> ProjectionResult<()> {
first_error(join_all(self.instances.values().map(ProjectionHandle::stop_and_wait)).await)
}
}
fn first_error(results: Vec<ProjectionResult<()>>) -> ProjectionResult<()> {
results
.into_iter()
.find_map(Result::err)
.map_or(Ok(()), Err)
}
#[cfg(test)]
mod tests {
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
time::Duration,
};
use tokio::sync::Semaphore;
use super::*;
use crate::{
PersistenceId, SequenceNr,
projection::{
Handler, MemoryProjectionOffsetStore, ProjectionOffsetStore, RecoveryStrategy,
RestartBackoff, SaveOffsetConfig, StatusObserver, TestEnvelope, TestSourceProvider,
},
query::QueryMode,
};
static NEXT_GROUP_TEST: AtomicU64 = AtomicU64::new(1);
type Envelope = TestEnvelope<i32>;
type Providers = BTreeMap<String, TestSourceProvider<i32>>;
type EventLog = BTreeMap<String, Vec<i32>>;
type FailurePlan = BTreeMap<(String, i32), usize>;
#[derive(Clone, Default)]
struct GroupProbes {
attempts: Arc<Mutex<EventLog>>,
completed: Arc<Mutex<EventLog>>,
failures: Arc<Mutex<FailurePlan>>,
factories: Arc<Mutex<BTreeMap<String, usize>>>,
restarts: Arc<Mutex<BTreeMap<String, usize>>>,
gate: Option<HandlerGate>,
}
#[derive(Clone)]
struct HandlerGate {
key: String,
payload: i32,
started: Arc<Semaphore>,
release: Arc<Semaphore>,
}
impl HandlerGate {
fn new(key: &str, payload: i32) -> Self {
Self {
key: key.to_owned(),
payload,
started: Arc::new(Semaphore::new(0)),
release: Arc::new(Semaphore::new(0)),
}
}
async fn wait_started(&self) {
Arc::clone(&self.started)
.acquire_owned()
.await
.expect("test handler gate remains open")
.forget();
}
fn release(&self) {
self.release.add_permits(1);
}
}
struct RecordingHandler {
key: String,
probes: GroupProbes,
}
#[async_trait::async_trait]
impl Handler<Envelope> for RecordingHandler {
async fn process(&mut self, envelope: &Envelope) -> ProjectionResult<()> {
push_log(&self.probes.attempts, &self.key, envelope.payload);
if let Some(gate) = &self.probes.gate
&& gate.key == self.key
&& gate.payload == envelope.payload
{
gate.started.add_permits(1);
Arc::clone(&gate.release)
.acquire_owned()
.await
.map_err(|_| ProjectionError::Handler("test handler gate closed".into()))?
.forget();
}
let should_fail = {
let mut failures = self.probes.failures.lock().expect("failure plan lock");
let remaining = failures
.entry((self.key.clone(), envelope.payload))
.or_default();
if *remaining > 0 {
*remaining -= 1;
true
} else {
false
}
};
if should_fail {
return Err(ProjectionError::Handler(format!(
"scripted failure for {}:{}",
self.key, envelope.payload
)));
}
push_log(&self.probes.completed, &self.key, envelope.payload);
Ok(())
}
}
#[derive(Clone)]
struct RestartObserver {
restarts: Arc<Mutex<BTreeMap<String, usize>>>,
}
impl StatusObserver<Envelope> for RestartObserver {
fn status_changed(&self, projection_id: &ProjectionId, status: &ProjectionStatus) {
if matches!(status, ProjectionStatus::Restarting { .. }) {
*self
.restarts
.lock()
.expect("restart observer lock")
.entry(projection_id.key().to_owned())
.or_default() += 1;
}
}
}
fn projection_group(
projection_name: &str,
keys: &[&str],
providers: Arc<Providers>,
probes: GroupProbes,
query_mode: QueryMode,
save_after_envelopes: usize,
restart_backoff: RestartBackoff,
) -> ProjectionGroup<Envelope> {
ProjectionGroup::for_keys(
projection_name,
keys.iter().copied(),
move |projection_id| {
let key = projection_id.key().to_owned();
let provider = providers
.get(&key)
.unwrap_or_else(|| panic!("missing test provider for {key}"))
.clone();
let handler_key = key.clone();
let handler_probes = probes.clone();
let observer = RestartObserver {
restarts: Arc::clone(&probes.restarts),
};
Projection::at_least_once(projection_id, provider, move || {
*handler_probes
.factories
.lock()
.expect("factory count lock")
.entry(handler_key.clone())
.or_default() += 1;
RecordingHandler {
key: handler_key.clone(),
probes: handler_probes.clone(),
}
})
.with_query_mode(query_mode)
.with_save_offset(SaveOffsetConfig {
after_envelopes: save_after_envelopes,
after_duration: Duration::from_secs(60),
})
.with_recovery_strategy(RecoveryStrategy::Fail)
.with_restart_backoff(restart_backoff)
.with_status_observer(observer)
},
)
}
fn provider(entries: &[(&str, u64, i32)]) -> TestSourceProvider<i32> {
TestSourceProvider::inclusive(
entries
.iter()
.map(|(key, position, payload)| envelope(key, *position, *payload))
.collect(),
)
}
fn envelope(key: &str, position: u64, payload: i32) -> Envelope {
TestEnvelope::new(
position,
PersistenceId::from_unique(format!("group-{key}|entity"))
.expect("test persistence id is valid"),
SequenceNr::new(position),
payload,
)
}
fn runtime(store: &Arc<MemoryProjectionOffsetStore>) -> ProjectionRuntime {
ProjectionRuntime::new(Arc::clone(store) as Arc<dyn ProjectionOffsetStore>)
}
fn no_delay_backoff() -> RestartBackoff {
RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: None,
max_restarts_within: Duration::from_secs(60),
}
}
fn test_name(label: &str) -> String {
format!(
"projection-group-{label}-{}",
NEXT_GROUP_TEST.fetch_add(1, Ordering::Relaxed)
)
}
fn push_log(log: &Mutex<EventLog>, key: &str, payload: i32) {
log.lock()
.expect("event log lock")
.entry(key.to_owned())
.or_default()
.push(payload);
}
fn logged(log: &Mutex<EventLog>, key: &str) -> Vec<i32> {
log.lock()
.expect("event log lock")
.get(key)
.cloned()
.unwrap_or_default()
}
fn count(counts: &Mutex<BTreeMap<String, usize>>, key: &str) -> usize {
counts
.lock()
.expect("count lock")
.get(key)
.copied()
.unwrap_or_default()
}
async fn wait_until(mut predicate: impl FnMut() -> bool) {
tokio::time::timeout(Duration::from_secs(5), async {
while !predicate() {
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("bounded group test condition timed out");
}
#[tokio::test]
async fn key_list_instances_run_concurrently_filter_by_key_and_keep_independent_cursors() {
let name = test_name("keys");
let keys = ["blue", "green", "red"];
let mut providers = Providers::new();
let mut paces = BTreeMap::new();
for (key, events) in [
("blue", vec![("blue", 1, 11), ("blue", 2, 12)]),
("green", vec![("green", 10, 21)]),
(
"red",
vec![("red", 100, 31), ("red", 200, 32), ("red", 300, 33)],
),
] {
let (provider, pace) = provider(&events).with_manual_pace();
providers.insert(key.to_owned(), provider);
paces.insert(key.to_owned(), pace);
}
let providers = Arc::new(providers);
let provider_probes = Arc::clone(&providers);
let probes = GroupProbes::default();
let group = projection_group(
&name,
&keys,
providers,
probes.clone(),
QueryMode::Current,
1,
no_delay_backoff(),
);
assert_eq!(group.len(), 3);
assert_eq!(group.projection_name(), name);
assert_eq!(
group
.projection_ids()
.map(ProjectionId::key)
.collect::<Vec<_>>(),
keys
);
assert!(
provider_probes
.values()
.all(|provider| provider.materializations().unwrap() == 0),
"building a group must not materialize any source"
);
assert!(
probes
.factories
.lock()
.expect("factory count lock")
.is_empty(),
"building a group must not construct handlers"
);
let store = Arc::new(MemoryProjectionOffsetStore::new());
let handle = group.spawn(runtime(&store)).await.unwrap();
wait_until(|| {
handle
.instances()
.all(|instance| instance.status().is_running())
})
.await;
assert_eq!(handle.statuses().len(), 3);
paces["blue"].request(2).unwrap();
paces["green"].request(1).unwrap();
paces["red"].request(3).unwrap();
for instance in handle.instances() {
assert_eq!(instance.wait().await.unwrap(), ProjectionStatus::Completed);
}
assert_eq!(logged(&probes.completed, "blue"), vec![11, 12]);
assert_eq!(logged(&probes.completed, "green"), vec![21]);
assert_eq!(logged(&probes.completed, "red"), vec![31, 32, 33]);
for key in keys {
assert!(
logged(&probes.completed, key)
.iter()
.all(|payload| match key {
"blue" => [11, 12].contains(payload),
"green" => [21].contains(payload),
"red" => [31, 32, 33].contains(payload),
_ => false,
}),
"instance {key} processed an envelope belonging to another key"
);
assert_eq!(count(&probes.factories, key), 1);
}
for (key, expected_position) in [("blue", "2"), ("green", "10"), ("red", "300")] {
let projection_id = ProjectionId::new(name.clone(), key);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("each current instance saves its own cursor")
.position()
.as_str(),
expected_position
);
assert_eq!(
handle
.instance(&projection_id)
.expect("complete id addresses an instance")
.projection_id(),
&projection_id
);
assert_eq!(
handle
.instance_by_key(key)
.expect("key addresses an instance")
.projection_id(),
&projection_id
);
}
assert!(
handle
.instance(&ProjectionId::new("another-name", "blue"))
.is_none()
);
}
#[tokio::test]
async fn handler_restart_is_isolated_from_live_sibling() {
let name = test_name("restart-isolation");
let (flaky_provider, flaky_pace) =
provider(&[("flaky", 1, 1), ("flaky", 2, 2)]).with_manual_pace();
let flaky_probe = flaky_provider.clone();
let (steady_provider, steady_pace) =
provider(&[("steady", 1, 10), ("steady", 2, 11)]).with_manual_pace();
let steady_probe = steady_provider.clone();
let providers = Arc::new(BTreeMap::from([
("flaky".to_owned(), flaky_provider),
("steady".to_owned(), steady_provider),
]));
let probes = GroupProbes::default();
probes
.failures
.lock()
.expect("failure plan lock")
.insert(("flaky".to_owned(), 1), 1);
let store = Arc::new(MemoryProjectionOffsetStore::new());
let handle = projection_group(
&name,
&["flaky", "steady"],
providers,
probes.clone(),
QueryMode::Live,
1,
no_delay_backoff(),
)
.spawn(runtime(&store))
.await
.unwrap();
wait_until(|| {
handle
.instances()
.all(|instance| instance.status().is_running())
})
.await;
flaky_pace.request(3).unwrap();
steady_pace.request_one().unwrap();
wait_until(|| {
count(&probes.restarts, "flaky") == 1
&& logged(&probes.completed, "flaky") == [1, 2]
&& logged(&probes.completed, "steady") == [10]
})
.await;
assert_eq!(flaky_probe.materializations().unwrap(), 2);
assert_eq!(count(&probes.factories, "flaky"), 2);
assert_eq!(count(&probes.restarts, "steady"), 0);
assert_eq!(steady_probe.materializations().unwrap(), 1);
assert_eq!(count(&probes.factories, "steady"), 1);
assert!(
handle
.instance_by_key("steady")
.unwrap()
.status()
.is_running()
);
steady_pace.request_one().unwrap();
wait_until(|| logged(&probes.completed, "steady") == [10, 11]).await;
assert_eq!(count(&probes.restarts, "steady"), 0);
assert_eq!(steady_probe.materializations().unwrap(), 1);
assert_eq!(logged(&probes.attempts, "flaky"), vec![1, 1, 2]);
handle.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn registry_rejects_cross_group_and_standalone_overlap_then_releases_on_stop() {
let group_name = test_name("registry-group");
let store = Arc::new(MemoryProjectionOffsetStore::new());
let first_providers = Arc::new(BTreeMap::from([
(
"a".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
(
"b".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
]));
let first = projection_group(
&group_name,
&["a", "b"],
first_providers,
GroupProbes::default(),
QueryMode::Live,
1,
no_delay_backoff(),
)
.spawn(runtime(&store))
.await
.unwrap();
wait_until(|| {
first
.instances()
.all(|instance| instance.status().is_running())
})
.await;
let overlap_b = TestSourceProvider::new(Vec::<Envelope>::new());
let overlap_c = TestSourceProvider::new(Vec::<Envelope>::new());
let overlap_probes = [overlap_b.clone(), overlap_c.clone()];
let cross_group = projection_group(
&group_name,
&["b", "c"],
Arc::new(BTreeMap::from([
("b".to_owned(), overlap_b),
("c".to_owned(), overlap_c),
])),
GroupProbes::default(),
QueryMode::Live,
1,
no_delay_backoff(),
)
.spawn(runtime(&store))
.await;
assert!(matches!(
cross_group,
Err(ProjectionError::AlreadyRunning { projection_id })
if projection_id == ProjectionId::new(group_name.clone(), "b")
));
assert!(
overlap_probes
.iter()
.all(|provider| provider.materializations().unwrap() == 0),
"atomic group reservation must reject overlap before any sibling starts"
);
let standalone_collision = Projection::at_least_once(
ProjectionId::new(group_name.clone(), "a"),
TestSourceProvider::new(Vec::<Envelope>::new()),
|| RecordingHandler {
key: "a".to_owned(),
probes: GroupProbes::default(),
},
)
.spawn(runtime(&store))
.await;
assert!(matches!(
standalone_collision,
Err(ProjectionError::AlreadyRunning { projection_id })
if projection_id == ProjectionId::new(group_name.clone(), "a")
));
let standalone_name = test_name("registry-standalone");
let standalone_id = ProjectionId::new(standalone_name.clone(), "x");
let standalone = Projection::at_least_once(
standalone_id.clone(),
TestSourceProvider::new(Vec::<Envelope>::new()),
|| RecordingHandler {
key: "x".to_owned(),
probes: GroupProbes::default(),
},
)
.spawn(runtime(&store))
.await
.unwrap();
let group_vs_standalone = projection_group(
&standalone_name,
&["x", "y"],
Arc::new(BTreeMap::from([
(
"x".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
(
"y".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
])),
GroupProbes::default(),
QueryMode::Live,
1,
no_delay_backoff(),
)
.spawn(runtime(&store))
.await;
assert!(matches!(
group_vs_standalone,
Err(ProjectionError::AlreadyRunning { projection_id })
if projection_id == standalone_id
));
standalone.stop_and_wait().await.unwrap();
first.stop_and_wait().await.unwrap();
let respawned = Projection::at_least_once(
ProjectionId::new(group_name.clone(), "a"),
TestSourceProvider::new(Vec::<Envelope>::new()),
|| RecordingHandler {
key: "a".to_owned(),
probes: GroupProbes::default(),
},
)
.spawn(runtime(&store))
.await
.expect("acknowledged group stop releases every registry reservation");
respawned.stop_and_wait().await.unwrap();
let replacement = projection_group(
&group_name,
&["a", "b"],
Arc::new(BTreeMap::from([
(
"a".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
(
"b".to_owned(),
TestSourceProvider::new(Vec::<Envelope>::new()),
),
])),
GroupProbes::default(),
QueryMode::Live,
1,
no_delay_backoff(),
)
.spawn(runtime(&store))
.await
.expect("all stopped group keys can be materialized again");
replacement.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn per_instance_and_group_pause_persist_and_group_stop_waits_for_every_flush() {
let name = test_name("management-stop");
let keys = ["a", "b"];
let (provider_a, pace_a) =
TestSourceProvider::inclusive(Vec::<Envelope>::new()).with_manual_pace();
let (provider_b, pace_b) =
TestSourceProvider::inclusive(Vec::<Envelope>::new()).with_manual_pace();
let providers = Arc::new(BTreeMap::from([
("a".to_owned(), provider_a.clone()),
("b".to_owned(), provider_b.clone()),
]));
let store = Arc::new(MemoryProjectionOffsetStore::new());
let probes = GroupProbes {
gate: Some(HandlerGate::new("b", 20)),
..GroupProbes::default()
};
let build = || {
projection_group(
&name,
&keys,
Arc::clone(&providers),
probes.clone(),
QueryMode::Live,
100,
no_delay_backoff(),
)
};
let id_a = ProjectionId::new(name.clone(), "a");
let id_b = ProjectionId::new(name.clone(), "b");
let first = build().spawn(runtime(&store)).await.unwrap();
wait_until(|| {
first
.instances()
.all(|instance| instance.status().is_running())
})
.await;
first.instance(&id_a).unwrap().pause().await.unwrap();
assert!(store.load_paused(&id_a).await.unwrap());
first.stop_and_wait().await.unwrap();
let second = build().spawn(runtime(&store)).await.unwrap();
wait_until(|| {
second.instance(&id_a).unwrap().status() == ProjectionStatus::Paused
&& second.instance(&id_b).unwrap().status().is_running()
})
.await;
second.pause_all().await.unwrap();
assert!(store.load_paused(&id_a).await.unwrap());
assert!(store.load_paused(&id_b).await.unwrap());
second.stop_and_wait().await.unwrap();
let third = build().spawn(runtime(&store)).await.unwrap();
wait_until(|| {
third
.instances()
.all(|instance| instance.status() == ProjectionStatus::Paused)
})
.await;
third.resume_all().await.unwrap();
wait_until(|| {
third
.instances()
.all(|instance| instance.status().is_running())
})
.await;
assert!(!store.load_paused(&id_a).await.unwrap());
assert!(!store.load_paused(&id_b).await.unwrap());
provider_a.push(envelope("a", 1, 10)).unwrap();
provider_b.push(envelope("b", 1, 20)).unwrap();
pace_a.request_one().unwrap();
pace_b.request_one().unwrap();
probes.gate.as_ref().unwrap().wait_started().await;
wait_until(|| logged(&probes.completed, "a").contains(&10)).await;
let instance_a = third.instance(&id_a).unwrap();
instance_a.pause().await.unwrap();
let saved_a = instance_a
.get_offset()
.await
.unwrap()
.expect("per-instance pause flushes its cursor");
instance_a.clear_offset().await.unwrap();
assert_eq!(instance_a.get_offset().await.unwrap(), None);
assert!(store.load_paused(&id_a).await.unwrap());
instance_a.update_offset(saved_a.clone()).await.unwrap();
assert_eq!(instance_a.get_offset().await.unwrap(), Some(saved_a));
instance_a.resume().await.unwrap();
let stop = third.stop_and_wait();
tokio::pin!(stop);
assert!(
tokio::time::timeout(Duration::from_millis(25), &mut stop)
.await
.is_err(),
"group stop must wait for a sibling's in-flight handler"
);
probes.gate.as_ref().unwrap().release();
stop.await.unwrap();
assert!(
third
.instances()
.all(|instance| instance.status() == ProjectionStatus::Stopped)
);
assert_eq!(
store
.load(&id_a)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"1"
);
assert_eq!(
store
.load(&id_b)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"1"
);
assert_eq!(logged(&probes.completed, "b"), vec![20]);
}
#[tokio::test]
async fn aborted_instances_resume_only_from_their_own_durable_cursors() {
let name = test_name("abort-resume");
let provider_a = provider(&[("a", 1, 1), ("a", 2, 2), ("a", 3, 3)]);
let provider_b = provider(&[("b", 1, 10), ("b", 2, 11)]);
let providers = Arc::new(BTreeMap::from([
("a".to_owned(), provider_a.clone()),
("b".to_owned(), provider_b.clone()),
]));
let probes = GroupProbes::default();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let build = || {
projection_group(
&name,
&["a", "b"],
Arc::clone(&providers),
probes.clone(),
QueryMode::Live,
2,
no_delay_backoff(),
)
};
let id_a = ProjectionId::new(name.clone(), "a");
let id_b = ProjectionId::new(name.clone(), "b");
let first = build().spawn(runtime(&store)).await.unwrap();
wait_until(|| {
logged(&probes.completed, "a") == [1, 2, 3]
&& logged(&probes.completed, "b") == [10, 11]
})
.await;
assert_eq!(
store
.load(&id_a)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"2",
"A has one unsaved at-least-once tail envelope"
);
assert_eq!(
store
.load(&id_b)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"2",
"B's independent cadence has fully committed"
);
for instance in first.instances() {
instance.abort_and_wait().await.unwrap();
}
drop(first);
provider_a.push(envelope("a", 4, 4)).unwrap();
provider_b.push(envelope("b", 3, 12)).unwrap();
let resumed = build().spawn(runtime(&store)).await.unwrap();
wait_until(|| {
logged(&probes.completed, "a") == [1, 2, 3, 3, 4]
&& logged(&probes.completed, "b") == [10, 11, 12]
})
.await;
resumed.stop_and_wait().await.unwrap();
assert_eq!(
logged(&probes.completed, "a"),
vec![1, 2, 3, 3, 4],
"A replays exactly its own one-envelope unsaved tail"
);
assert_eq!(
logged(&probes.completed, "b"),
vec![10, 11, 12],
"B neither receives A's handoff nor replays its committed cursor"
);
assert_eq!(
store
.load(&id_a)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"4"
);
assert_eq!(
store
.load(&id_b)
.await
.unwrap()
.unwrap()
.position()
.as_str(),
"3"
);
}
#[tokio::test]
async fn invalid_key_sets_and_template_identity_fail_before_materialization() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let name = test_name("invalid");
let duplicate_provider = TestSourceProvider::new(Vec::<Envelope>::new());
let duplicate_probe = duplicate_provider.clone();
let duplicate = ProjectionGroup::for_keys(&name, ["same", "same"], move |projection_id| {
Projection::at_least_once(projection_id, duplicate_provider.clone(), || {
RecordingHandler {
key: "same".to_owned(),
probes: GroupProbes::default(),
}
})
})
.spawn(runtime(&store))
.await;
assert!(matches!(
duplicate,
Err(ProjectionError::InvalidConfig {
setting: "projection_group.keys",
..
})
));
assert_eq!(duplicate_probe.materializations().unwrap(), 0);
let mismatched_provider = TestSourceProvider::new(Vec::<Envelope>::new());
let mismatch_probe = mismatched_provider.clone();
let mismatch = ProjectionGroup::for_keys(&name, ["expected"], move |_projection_id| {
Projection::at_least_once(
ProjectionId::new("wrong", "identity"),
mismatched_provider.clone(),
|| RecordingHandler {
key: "wrong".to_owned(),
probes: GroupProbes::default(),
},
)
})
.spawn(runtime(&store))
.await;
assert!(matches!(
mismatch,
Err(ProjectionError::InvalidConfig {
setting: "projection_group.template.projection_id",
..
})
));
assert_eq!(mismatch_probe.materializations().unwrap(), 0);
}
}