1use std::io;
11
12use serde_json::Value;
13use taskfast_agent::events::{decode_event_item, TolerantEventPage, UnparseableEvent};
14use taskfast_client::api::types::{AgentEvent, PaginatedMeta};
15
16use super::seen::SeenIds;
17use crate::cmd::Ctx;
18use crate::state_store::StateStore;
19
20pub const DISABLE_SENTINEL: &str = "-";
22
23pub const DISABLE_ENV: &str = "TASKFAST_NO_CURSOR_STATE";
26
27#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
29pub struct IntakeMetrics {
30 pub total_received: u64,
32 pub accepted_events: u64,
34 pub deduplicated_events: u64,
36 pub unparseable_events: u64,
38}
39
40#[derive(Debug, Clone)]
42pub struct IngestBatchOutcome {
43 pub accepted_events: Vec<AgentEvent>,
44 pub unparseable_events: Vec<UnparseableEvent>,
45 pub meta: PaginatedMeta,
46}
47
48#[derive(Debug, Clone)]
50pub enum StreamItemOutcome {
51 Accepted { event_id: String },
52 Duplicate { event_id: String },
53 Malformed { error: String },
54}
55
56#[derive(Debug, Clone, Default, PartialEq, Eq)]
57enum PendingCursor {
58 #[default]
59 None,
60 Update(Option<String>),
61}
62
63#[allow(clippy::struct_excessive_bools)]
65pub struct EventIntake {
66 store: StateStore,
67 seen: SeenIds,
68 request_cursor: Option<String>,
69 persistence_enabled: bool,
70 replay_mode: bool,
71 metrics: IntakeMetrics,
72 pending_seen: Vec<String>,
73 pending_cursor: PendingCursor,
74 quiet: bool,
75 warned: bool,
76}
77
78impl EventIntake {
79 pub fn from_ctx(ctx: &Ctx, cursor_arg: Option<&str>) -> Self {
81 let store = ctx.state_store();
82 Self::new(store, cursor_arg, ctx.quiet)
83 }
84
85 pub fn new(store: StateStore, cursor_arg: Option<&str>, quiet: bool) -> Self {
87 let env_disabled = std::env::var(DISABLE_ENV).is_ok_and(|v| v == "1");
88 let sentinel = matches!(cursor_arg, Some(s) if s == DISABLE_SENTINEL);
89 let persistence_enabled = !env_disabled && !sentinel && !store.is_ephemeral();
90
91 let replay_mode = cursor_arg.is_some();
92
93 let request_cursor = match cursor_arg {
94 Some(s) if s == DISABLE_SENTINEL => None,
95 Some(s) => Some(s.to_string()),
96 None => {
97 if persistence_enabled {
98 store.read_cursor()
99 } else {
100 None
101 }
102 }
103 };
104
105 let seen = SeenIds::new(store.clone(), quiet);
106
107 Self {
108 store,
109 seen,
110 request_cursor,
111 persistence_enabled,
112 replay_mode,
113 metrics: IntakeMetrics::default(),
114 pending_seen: Vec::new(),
115 pending_cursor: PendingCursor::None,
116 quiet,
117 warned: false,
118 }
119 }
120
121 pub fn effective_cursor(&self) -> Option<&str> {
123 self.request_cursor.as_deref()
124 }
125
126 pub fn persistence_enabled(&self) -> bool {
128 self.persistence_enabled
129 }
130
131 pub fn replay_mode(&self) -> bool {
133 self.replay_mode
134 }
135
136 pub fn metrics(&self) -> IntakeMetrics {
138 self.metrics
139 }
140
141 pub fn ingest_poll_page(&mut self, page: TolerantEventPage) -> IngestBatchOutcome {
143 self.ingest_batch(page.events, page.unparseable, page.meta)
144 }
145
146 pub fn ingest_batch(
148 &mut self,
149 events: Vec<AgentEvent>,
150 unparseable: Vec<UnparseableEvent>,
151 meta: PaginatedMeta,
152 ) -> IngestBatchOutcome {
153 for u in &unparseable {
154 self.metrics.unparseable_events += 1;
155 tracing::warn!(
156 target: "taskfast::events",
157 raw = %u.raw,
158 error = %u.error,
159 "unparseable event surfaced via tolerant decode"
160 );
161 }
162
163 let mut accepted = Vec::new();
164 for ev in events {
165 self.metrics.total_received += 1;
166 let id = ev.event_id.to_string();
167 let is_duplicate = self.seen.contains(&id);
168
169 if is_duplicate && !self.replay_mode {
170 self.metrics.deduplicated_events += 1;
171 } else {
172 self.metrics.accepted_events += 1;
173 self.seen.remember(&id);
174 self.pending_seen.push(id);
175 accepted.push(ev);
176 }
177 }
178
179 self.pending_cursor = PendingCursor::Update(meta.next_cursor.clone());
180
181 IngestBatchOutcome {
182 accepted_events: accepted,
183 unparseable_events: unparseable,
184 meta,
185 }
186 }
187
188 pub fn ingest_stream_payload(&mut self, payload: &Value) -> StreamItemOutcome {
190 self.metrics.total_received += 1;
191 let Some(event_id) = payload.get("event_id").and_then(Value::as_str) else {
192 self.metrics.unparseable_events += 1;
193 return StreamItemOutcome::Malformed {
194 error: "missing event_id in event payload".to_string(),
195 };
196 };
197
198 if self.seen.contains(event_id) {
199 self.metrics.deduplicated_events += 1;
200 return StreamItemOutcome::Duplicate {
201 event_id: event_id.to_string(),
202 };
203 }
204
205 if let Err(u) = decode_event_item(payload.clone()) {
206 self.metrics.unparseable_events += 1;
207 tracing::warn!(
208 target: "taskfast::events",
209 event_id = %event_id,
210 error = %u.error,
211 "schema drift in lifecycle event payload"
212 );
213 } else {
214 self.metrics.accepted_events += 1;
215 }
216
217 if self.persistence_enabled {
218 self.seen.commit_ids(&[event_id.to_string()]);
219 } else {
220 self.seen.remember(event_id);
221 }
222
223 StreamItemOutcome::Accepted {
224 event_id: event_id.to_string(),
225 }
226 }
227
228 pub fn record_unparseable(&mut self) {
230 self.metrics.unparseable_events += 1;
231 }
232
233 pub fn commit(&mut self) -> io::Result<()> {
235 if self.persistence_enabled {
236 if let PendingCursor::Update(ref cursor_opt) = self.pending_cursor {
237 self.store.write_cursor(cursor_opt.as_deref())?;
238 }
239 self.pending_cursor = PendingCursor::None;
240 if !self.pending_seen.is_empty() {
241 self.seen.commit_ids(&self.pending_seen);
242 self.pending_seen.clear();
243 }
244 } else {
245 for id in self.pending_seen.drain(..) {
246 self.seen.remember(&id);
247 }
248 self.pending_cursor = PendingCursor::None;
249 }
250 Ok(())
251 }
252
253 pub fn warn(&mut self, msg: &str) {
255 if self.warned || self.quiet {
256 return;
257 }
258 self.warned = true;
259 eprintln!("warning: {msg}");
260 }
261}
262
263#[cfg(test)]
264mod tests {
265 use super::*;
266 use serde_json::json;
267 use tempfile::TempDir;
268
269 fn sample_event(id: &str, name: &str) -> AgentEvent {
270 serde_json::from_value(json!({
271 "event_id": id,
272 "event": name,
273 "occurred_at": "2026-11-15T22:00:00Z",
274 "data": {},
275 }))
276 .unwrap()
277 }
278
279 #[test]
280 fn in_memory_cursor_and_seen_deduplication() {
281 let tmp = TempDir::new().unwrap();
282 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
283 let mut intake = EventIntake::new(store.clone(), None, true);
284
285 assert_eq!(intake.effective_cursor(), None);
286
287 let ev1 = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
288 let ev2 = sample_event("00000000-0000-0000-0000-000000000002", "bid_placed");
289
290 let outcome = intake.ingest_batch(
291 vec![ev1.clone(), ev2.clone()],
292 Vec::new(),
293 PaginatedMeta {
294 has_more: true,
295 next_cursor: Some("cur_1".to_string()),
296 total_count: Some(2),
297 },
298 );
299
300 assert_eq!(outcome.accepted_events.len(), 2);
301 assert_eq!(store.read_cursor(), None);
302
303 intake.commit().unwrap();
304 assert_eq!(store.read_cursor(), Some("cur_1".to_string()));
305
306 let ev3 = sample_event("00000000-0000-0000-0000-000000000003", "task_completed");
307 let outcome2 = intake.ingest_batch(
308 vec![ev2.clone(), ev3.clone()],
309 Vec::new(),
310 PaginatedMeta {
311 has_more: false,
312 next_cursor: Some("cur_2".to_string()),
313 total_count: Some(2),
314 },
315 );
316
317 assert_eq!(outcome2.accepted_events.len(), 1);
318 assert_eq!(outcome2.accepted_events[0].event_id, ev3.event_id);
319 assert_eq!(intake.metrics().deduplicated_events, 1);
320
321 intake.commit().unwrap();
322 assert_eq!(store.read_cursor(), Some("cur_2".to_string()));
323
324 let mut intake_reconnect = EventIntake::new(store, None, true);
325 assert_eq!(intake_reconnect.effective_cursor(), Some("cur_2"));
326
327 let outcome3 = intake_reconnect.ingest_batch(
328 vec![ev3],
329 Vec::new(),
330 PaginatedMeta {
331 has_more: false,
332 next_cursor: None,
333 total_count: Some(1),
334 },
335 );
336 assert_eq!(outcome3.accepted_events.len(), 0);
337 assert_eq!(intake_reconnect.metrics().deduplicated_events, 1);
338 }
339
340 #[test]
341 fn within_batch_duplicate_events_are_deduplicated() {
342 let store = StateStore::ephemeral();
343 let mut intake = EventIntake::new(store, None, true);
344
345 let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
346 let outcome = intake.ingest_batch(
347 vec![ev.clone(), ev],
348 Vec::new(),
349 PaginatedMeta {
350 has_more: false,
351 next_cursor: None,
352 total_count: Some(2),
353 },
354 );
355
356 assert_eq!(outcome.accepted_events.len(), 1);
357 assert_eq!(intake.metrics().deduplicated_events, 1);
358 assert_eq!(intake.metrics().accepted_events, 1);
359 }
360
361 #[test]
362 fn dash_sentinel_replays_all_seen_events() {
363 let tmp = TempDir::new().unwrap();
364 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
365
366 let mut init_intake = EventIntake::new(store.clone(), None, true);
367 let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
368 init_intake.ingest_batch(
369 vec![ev.clone()],
370 Vec::new(),
371 PaginatedMeta {
372 has_more: false,
373 next_cursor: Some("c1".to_string()),
374 total_count: Some(1),
375 },
376 );
377 init_intake.commit().unwrap();
378
379 let mut sentinel_intake = EventIntake::new(store, Some(DISABLE_SENTINEL), true);
381 assert_eq!(sentinel_intake.effective_cursor(), None);
382 assert!(!sentinel_intake.persistence_enabled());
383 assert!(sentinel_intake.replay_mode());
384
385 let outcome = sentinel_intake.ingest_batch(
386 vec![ev],
387 Vec::new(),
388 PaginatedMeta {
389 has_more: false,
390 next_cursor: None,
391 total_count: Some(1),
392 },
393 );
394 assert_eq!(
395 outcome.accepted_events.len(),
396 1,
397 "sentinel must replay seen events"
398 );
399 }
400
401 #[test]
402 fn uncommitted_batch_preserves_previous_state() {
403 let tmp = TempDir::new().unwrap();
404 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
405 store.write_cursor(Some("initial_cursor")).unwrap();
406
407 let mut intake = EventIntake::new(store.clone(), None, true);
408 assert_eq!(intake.effective_cursor(), Some("initial_cursor"));
409
410 let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
411 let _ = intake.ingest_batch(
412 vec![ev],
413 Vec::new(),
414 PaginatedMeta {
415 has_more: false,
416 next_cursor: Some("aborted_cursor".to_string()),
417 total_count: Some(1),
418 },
419 );
420
421 drop(intake);
422
423 let resumed = EventIntake::new(store, None, true);
424 assert_eq!(resumed.effective_cursor(), Some("initial_cursor"));
425 assert!(!resumed
426 .seen
427 .contains("00000000-0000-0000-0000-000000000001"));
428 }
429
430 #[test]
431 fn stream_payload_deduplication_and_drift_handling() {
432 let tmp = TempDir::new().unwrap();
433 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
434 let mut intake = EventIntake::new(store, None, true);
435
436 let payload = json!({
437 "event_id": "00000000-0000-0000-0000-000000000001",
438 "event": "task_assigned",
439 "timestamp": "2026-11-15T22:00:00Z",
441 "data": {},
442 });
443
444 match intake.ingest_stream_payload(&payload) {
445 StreamItemOutcome::Accepted { event_id } => {
446 assert_eq!(event_id, "00000000-0000-0000-0000-000000000001");
447 }
448 other => panic!("expected Accepted, got {other:?}"),
449 }
450
451 assert_eq!(intake.metrics().unparseable_events, 1);
452 assert_eq!(intake.metrics().accepted_events, 0);
453
454 match intake.ingest_stream_payload(&payload) {
455 StreamItemOutcome::Duplicate { event_id } => {
456 assert_eq!(event_id, "00000000-0000-0000-0000-000000000001");
457 }
458 other => panic!("expected Duplicate, got {other:?}"),
459 }
460
461 assert_eq!(intake.metrics().deduplicated_events, 1);
462 }
463
464 #[test]
465 fn dash_sentinel_disables_persistence() {
466 let tmp = TempDir::new().unwrap();
467 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
468 let intake = EventIntake::new(store, Some(DISABLE_SENTINEL), true);
469
470 assert!(!intake.persistence_enabled());
471 assert_eq!(intake.effective_cursor(), None);
472 }
473
474 #[test]
475 fn replay_mode_accepts_duplicates_and_records_ids() {
476 let store = StateStore::ephemeral();
477 let mut intake = EventIntake::new(store, Some("explicit_replay_cursor"), true);
478 assert!(intake.replay_mode());
479
480 let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
481
482 let o1 = intake.ingest_batch(
483 vec![ev.clone()],
484 Vec::new(),
485 PaginatedMeta {
486 has_more: false,
487 next_cursor: None,
488 total_count: Some(1),
489 },
490 );
491 assert_eq!(o1.accepted_events.len(), 1);
492
493 intake.commit().unwrap();
494
495 let o2 = intake.ingest_batch(
496 vec![ev],
497 Vec::new(),
498 PaginatedMeta {
499 has_more: false,
500 next_cursor: None,
501 total_count: Some(1),
502 },
503 );
504 assert_eq!(o2.accepted_events.len(), 1);
505 }
506
507 #[test]
508 fn commit_cursor_write_failure_preserves_pending_state() {
509 let tmp = TempDir::new().unwrap();
510 let store = StateStore::from_config_path(&tmp.path().join("config.json"));
511 std::fs::create_dir_all(
513 store
514 .path_for(crate::state_store::StateAsset::EventsCursor)
515 .unwrap(),
516 )
517 .unwrap();
518
519 let mut intake = EventIntake::new(store, None, true);
520 let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
521 intake.ingest_batch(
522 vec![ev],
523 Vec::new(),
524 PaginatedMeta {
525 has_more: false,
526 next_cursor: Some("c_fail".to_string()),
527 total_count: Some(1),
528 },
529 );
530
531 assert!(intake.commit().is_err());
532 assert_eq!(intake.pending_seen.len(), 1);
533 assert!(matches!(intake.pending_cursor, PendingCursor::Update(_)));
534 }
535}