Skip to main content

taskfast_cli/cmd/events/
intake.rs

1// SPDX-License-Identifier: MIT
2//! Deep lifecycle event intake module (gh#217).
3//!
4//! Encapsulates:
5//! - Tolerant per-item JSON decoding with metrics/counters for schema drift.
6//! - Bounded [`SeenIds`] ring-buffer deduplication across reconnects and poll iterations.
7//! - Coordinated cursor progression and seen-id commitment after successful batch processing (integrating with [`StateStore`]).
8//! - A clean seam accepting incoming event batches from both HTTP polling and WebSocket streaming transports.
9
10use std::io;
11
12use serde_json::Value;
13use taskfast_agent::events::{decode_event_item, TolerantEventPage, UnparseableEvent};
14use taskfast_client::api::types::{AgentEvent, PaginatedMeta};
15
16use super::seen::SeenIds;
17use crate::cmd::Ctx;
18use crate::state_store::StateStore;
19
20/// Sentinel passed via `--cursor -` to bypass the file completely.
21pub const DISABLE_SENTINEL: &str = "-";
22
23/// Env var that disables cursor persistence globally (read + write). Useful
24/// for CI or transient debug runs that share a config dir with a real loop.
25pub const DISABLE_ENV: &str = "TASKFAST_NO_CURSOR_STATE";
26
27/// Metrics and counters for schema drift and event intake.
28#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
29pub struct IntakeMetrics {
30    /// Total events received across poll and stream intakes.
31    pub total_received: u64,
32    /// Successfully accepted events.
33    pub accepted_events: u64,
34    /// Events dropped due to ring-buffer deduplication.
35    pub deduplicated_events: u64,
36    /// Events that failed item-level decoding due to schema drift.
37    pub unparseable_events: u64,
38}
39
40/// Outcome of ingesting a batch of events.
41#[derive(Debug, Clone)]
42pub struct IngestBatchOutcome {
43    pub accepted_events: Vec<AgentEvent>,
44    pub unparseable_events: Vec<UnparseableEvent>,
45    pub meta: PaginatedMeta,
46}
47
48/// Outcome of ingesting a single stream payload.
49#[derive(Debug, Clone)]
50pub enum StreamItemOutcome {
51    Accepted { event_id: String },
52    Duplicate { event_id: String },
53    Malformed { error: String },
54}
55
56#[derive(Debug, Clone, Default, PartialEq, Eq)]
57enum PendingCursor {
58    #[default]
59    None,
60    Update(Option<String>),
61}
62
63/// Unified event intake orchestrator.
64#[allow(clippy::struct_excessive_bools)]
65pub struct EventIntake {
66    store: StateStore,
67    seen: SeenIds,
68    request_cursor: Option<String>,
69    persistence_enabled: bool,
70    replay_mode: bool,
71    metrics: IntakeMetrics,
72    pending_seen: Vec<String>,
73    pending_cursor: PendingCursor,
74    quiet: bool,
75    warned: bool,
76}
77
78impl EventIntake {
79    /// Construct an intake instance from CLI context and optional `--cursor` flag.
80    pub fn from_ctx(ctx: &Ctx, cursor_arg: Option<&str>) -> Self {
81        let store = ctx.state_store();
82        Self::new(store, cursor_arg, ctx.quiet)
83    }
84
85    /// Construct an intake instance with explicit store, cursor argument, and quiet flag.
86    pub fn new(store: StateStore, cursor_arg: Option<&str>, quiet: bool) -> Self {
87        let env_disabled = std::env::var(DISABLE_ENV).is_ok_and(|v| v == "1");
88        let sentinel = matches!(cursor_arg, Some(s) if s == DISABLE_SENTINEL);
89        let persistence_enabled = !env_disabled && !sentinel && !store.is_ephemeral();
90
91        let replay_mode = cursor_arg.is_some();
92
93        let request_cursor = match cursor_arg {
94            Some(s) if s == DISABLE_SENTINEL => None,
95            Some(s) => Some(s.to_string()),
96            None => {
97                if persistence_enabled {
98                    store.read_cursor()
99                } else {
100                    None
101                }
102            }
103        };
104
105        let seen = SeenIds::new(store.clone(), quiet);
106
107        Self {
108            store,
109            seen,
110            request_cursor,
111            persistence_enabled,
112            replay_mode,
113            metrics: IntakeMetrics::default(),
114            pending_seen: Vec::new(),
115            pending_cursor: PendingCursor::None,
116            quiet,
117            warned: false,
118        }
119    }
120
121    /// Effective cursor to use for request pagination.
122    pub fn effective_cursor(&self) -> Option<&str> {
123        self.request_cursor.as_deref()
124    }
125
126    /// Whether cursor and seen persistence are active.
127    pub fn persistence_enabled(&self) -> bool {
128        self.persistence_enabled
129    }
130
131    /// Whether intake is operating in replay mode (explicit cursor passed).
132    pub fn replay_mode(&self) -> bool {
133        self.replay_mode
134    }
135
136    /// Snapshot of intake metrics.
137    pub fn metrics(&self) -> IntakeMetrics {
138        self.metrics
139    }
140
141    /// Ingest a tolerant event page from an HTTP poll response.
142    pub fn ingest_poll_page(&mut self, page: TolerantEventPage) -> IngestBatchOutcome {
143        self.ingest_batch(page.events, page.unparseable, page.meta)
144    }
145
146    /// Ingest a batch of typed events, unparseable items, and pagination metadata.
147    pub fn ingest_batch(
148        &mut self,
149        events: Vec<AgentEvent>,
150        unparseable: Vec<UnparseableEvent>,
151        meta: PaginatedMeta,
152    ) -> IngestBatchOutcome {
153        for u in &unparseable {
154            self.metrics.unparseable_events += 1;
155            tracing::warn!(
156                target: "taskfast::events",
157                raw = %u.raw,
158                error = %u.error,
159                "unparseable event surfaced via tolerant decode"
160            );
161        }
162
163        let mut accepted = Vec::new();
164        for ev in events {
165            self.metrics.total_received += 1;
166            let id = ev.event_id.to_string();
167            let is_duplicate = self.seen.contains(&id);
168
169            if is_duplicate && !self.replay_mode {
170                self.metrics.deduplicated_events += 1;
171            } else {
172                self.metrics.accepted_events += 1;
173                self.seen.remember(&id);
174                self.pending_seen.push(id);
175                accepted.push(ev);
176            }
177        }
178
179        self.pending_cursor = PendingCursor::Update(meta.next_cursor.clone());
180
181        IngestBatchOutcome {
182            accepted_events: accepted,
183            unparseable_events: unparseable,
184            meta,
185        }
186    }
187
188    /// Ingest a single WebSocket event payload.
189    pub fn ingest_stream_payload(&mut self, payload: &Value) -> StreamItemOutcome {
190        self.metrics.total_received += 1;
191        let Some(event_id) = payload.get("event_id").and_then(Value::as_str) else {
192            self.metrics.unparseable_events += 1;
193            return StreamItemOutcome::Malformed {
194                error: "missing event_id in event payload".to_string(),
195            };
196        };
197
198        if self.seen.contains(event_id) {
199            self.metrics.deduplicated_events += 1;
200            return StreamItemOutcome::Duplicate {
201                event_id: event_id.to_string(),
202            };
203        }
204
205        if let Err(u) = decode_event_item(payload.clone()) {
206            self.metrics.unparseable_events += 1;
207            tracing::warn!(
208                target: "taskfast::events",
209                event_id = %event_id,
210                error = %u.error,
211                "schema drift in lifecycle event payload"
212            );
213        } else {
214            self.metrics.accepted_events += 1;
215        }
216
217        if self.persistence_enabled {
218            self.seen.commit_ids(&[event_id.to_string()]);
219        } else {
220            self.seen.remember(event_id);
221        }
222
223        StreamItemOutcome::Accepted {
224            event_id: event_id.to_string(),
225        }
226    }
227
228    /// Record an unparseable non-JSON text frame.
229    pub fn record_unparseable(&mut self) {
230        self.metrics.unparseable_events += 1;
231    }
232
233    /// Commit cursor progression and seen-id deduplication state to StateStore.
234    pub fn commit(&mut self) -> io::Result<()> {
235        if self.persistence_enabled {
236            if let PendingCursor::Update(ref cursor_opt) = self.pending_cursor {
237                self.store.write_cursor(cursor_opt.as_deref())?;
238            }
239            self.pending_cursor = PendingCursor::None;
240            if !self.pending_seen.is_empty() {
241                self.seen.commit_ids(&self.pending_seen);
242                self.pending_seen.clear();
243            }
244        } else {
245            for id in self.pending_seen.drain(..) {
246                self.seen.remember(&id);
247            }
248            self.pending_cursor = PendingCursor::None;
249        }
250        Ok(())
251    }
252
253    /// Print a single warning to stderr if not in quiet mode.
254    pub fn warn(&mut self, msg: &str) {
255        if self.warned || self.quiet {
256            return;
257        }
258        self.warned = true;
259        eprintln!("warning: {msg}");
260    }
261}
262
263#[cfg(test)]
264mod tests {
265    use super::*;
266    use serde_json::json;
267    use tempfile::TempDir;
268
269    fn sample_event(id: &str, name: &str) -> AgentEvent {
270        serde_json::from_value(json!({
271            "event_id": id,
272            "event": name,
273            "occurred_at": "2026-11-15T22:00:00Z",
274            "data": {},
275        }))
276        .unwrap()
277    }
278
279    #[test]
280    fn in_memory_cursor_and_seen_deduplication() {
281        let tmp = TempDir::new().unwrap();
282        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
283        let mut intake = EventIntake::new(store.clone(), None, true);
284
285        assert_eq!(intake.effective_cursor(), None);
286
287        let ev1 = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
288        let ev2 = sample_event("00000000-0000-0000-0000-000000000002", "bid_placed");
289
290        let outcome = intake.ingest_batch(
291            vec![ev1.clone(), ev2.clone()],
292            Vec::new(),
293            PaginatedMeta {
294                has_more: true,
295                next_cursor: Some("cur_1".to_string()),
296                total_count: Some(2),
297            },
298        );
299
300        assert_eq!(outcome.accepted_events.len(), 2);
301        assert_eq!(store.read_cursor(), None);
302
303        intake.commit().unwrap();
304        assert_eq!(store.read_cursor(), Some("cur_1".to_string()));
305
306        let ev3 = sample_event("00000000-0000-0000-0000-000000000003", "task_completed");
307        let outcome2 = intake.ingest_batch(
308            vec![ev2.clone(), ev3.clone()],
309            Vec::new(),
310            PaginatedMeta {
311                has_more: false,
312                next_cursor: Some("cur_2".to_string()),
313                total_count: Some(2),
314            },
315        );
316
317        assert_eq!(outcome2.accepted_events.len(), 1);
318        assert_eq!(outcome2.accepted_events[0].event_id, ev3.event_id);
319        assert_eq!(intake.metrics().deduplicated_events, 1);
320
321        intake.commit().unwrap();
322        assert_eq!(store.read_cursor(), Some("cur_2".to_string()));
323
324        let mut intake_reconnect = EventIntake::new(store, None, true);
325        assert_eq!(intake_reconnect.effective_cursor(), Some("cur_2"));
326
327        let outcome3 = intake_reconnect.ingest_batch(
328            vec![ev3],
329            Vec::new(),
330            PaginatedMeta {
331                has_more: false,
332                next_cursor: None,
333                total_count: Some(1),
334            },
335        );
336        assert_eq!(outcome3.accepted_events.len(), 0);
337        assert_eq!(intake_reconnect.metrics().deduplicated_events, 1);
338    }
339
340    #[test]
341    fn within_batch_duplicate_events_are_deduplicated() {
342        let store = StateStore::ephemeral();
343        let mut intake = EventIntake::new(store, None, true);
344
345        let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
346        let outcome = intake.ingest_batch(
347            vec![ev.clone(), ev],
348            Vec::new(),
349            PaginatedMeta {
350                has_more: false,
351                next_cursor: None,
352                total_count: Some(2),
353            },
354        );
355
356        assert_eq!(outcome.accepted_events.len(), 1);
357        assert_eq!(intake.metrics().deduplicated_events, 1);
358        assert_eq!(intake.metrics().accepted_events, 1);
359    }
360
361    #[test]
362    fn dash_sentinel_replays_all_seen_events() {
363        let tmp = TempDir::new().unwrap();
364        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
365
366        let mut init_intake = EventIntake::new(store.clone(), None, true);
367        let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
368        init_intake.ingest_batch(
369            vec![ev.clone()],
370            Vec::new(),
371            PaginatedMeta {
372                has_more: false,
373                next_cursor: Some("c1".to_string()),
374                total_count: Some(1),
375            },
376        );
377        init_intake.commit().unwrap();
378
379        // One-shot inspection with `--cursor -` against populated seen-file
380        let mut sentinel_intake = EventIntake::new(store, Some(DISABLE_SENTINEL), true);
381        assert_eq!(sentinel_intake.effective_cursor(), None);
382        assert!(!sentinel_intake.persistence_enabled());
383        assert!(sentinel_intake.replay_mode());
384
385        let outcome = sentinel_intake.ingest_batch(
386            vec![ev],
387            Vec::new(),
388            PaginatedMeta {
389                has_more: false,
390                next_cursor: None,
391                total_count: Some(1),
392            },
393        );
394        assert_eq!(
395            outcome.accepted_events.len(),
396            1,
397            "sentinel must replay seen events"
398        );
399    }
400
401    #[test]
402    fn uncommitted_batch_preserves_previous_state() {
403        let tmp = TempDir::new().unwrap();
404        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
405        store.write_cursor(Some("initial_cursor")).unwrap();
406
407        let mut intake = EventIntake::new(store.clone(), None, true);
408        assert_eq!(intake.effective_cursor(), Some("initial_cursor"));
409
410        let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
411        let _ = intake.ingest_batch(
412            vec![ev],
413            Vec::new(),
414            PaginatedMeta {
415                has_more: false,
416                next_cursor: Some("aborted_cursor".to_string()),
417                total_count: Some(1),
418            },
419        );
420
421        drop(intake);
422
423        let resumed = EventIntake::new(store, None, true);
424        assert_eq!(resumed.effective_cursor(), Some("initial_cursor"));
425        assert!(!resumed
426            .seen
427            .contains("00000000-0000-0000-0000-000000000001"));
428    }
429
430    #[test]
431    fn stream_payload_deduplication_and_drift_handling() {
432        let tmp = TempDir::new().unwrap();
433        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
434        let mut intake = EventIntake::new(store, None, true);
435
436        let payload = json!({
437            "event_id": "00000000-0000-0000-0000-000000000001",
438            "event": "task_assigned",
439            // Has timestamp instead of occurred_at (schema drift)
440            "timestamp": "2026-11-15T22:00:00Z",
441            "data": {},
442        });
443
444        match intake.ingest_stream_payload(&payload) {
445            StreamItemOutcome::Accepted { event_id } => {
446                assert_eq!(event_id, "00000000-0000-0000-0000-000000000001");
447            }
448            other => panic!("expected Accepted, got {other:?}"),
449        }
450
451        assert_eq!(intake.metrics().unparseable_events, 1);
452        assert_eq!(intake.metrics().accepted_events, 0);
453
454        match intake.ingest_stream_payload(&payload) {
455            StreamItemOutcome::Duplicate { event_id } => {
456                assert_eq!(event_id, "00000000-0000-0000-0000-000000000001");
457            }
458            other => panic!("expected Duplicate, got {other:?}"),
459        }
460
461        assert_eq!(intake.metrics().deduplicated_events, 1);
462    }
463
464    #[test]
465    fn dash_sentinel_disables_persistence() {
466        let tmp = TempDir::new().unwrap();
467        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
468        let intake = EventIntake::new(store, Some(DISABLE_SENTINEL), true);
469
470        assert!(!intake.persistence_enabled());
471        assert_eq!(intake.effective_cursor(), None);
472    }
473
474    #[test]
475    fn replay_mode_accepts_duplicates_and_records_ids() {
476        let store = StateStore::ephemeral();
477        let mut intake = EventIntake::new(store, Some("explicit_replay_cursor"), true);
478        assert!(intake.replay_mode());
479
480        let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
481
482        let o1 = intake.ingest_batch(
483            vec![ev.clone()],
484            Vec::new(),
485            PaginatedMeta {
486                has_more: false,
487                next_cursor: None,
488                total_count: Some(1),
489            },
490        );
491        assert_eq!(o1.accepted_events.len(), 1);
492
493        intake.commit().unwrap();
494
495        let o2 = intake.ingest_batch(
496            vec![ev],
497            Vec::new(),
498            PaginatedMeta {
499                has_more: false,
500                next_cursor: None,
501                total_count: Some(1),
502            },
503        );
504        assert_eq!(o2.accepted_events.len(), 1);
505    }
506
507    #[test]
508    fn commit_cursor_write_failure_preserves_pending_state() {
509        let tmp = TempDir::new().unwrap();
510        let store = StateStore::from_config_path(&tmp.path().join("config.json"));
511        // Make the cursor path a directory so writing fails
512        std::fs::create_dir_all(
513            store
514                .path_for(crate::state_store::StateAsset::EventsCursor)
515                .unwrap(),
516        )
517        .unwrap();
518
519        let mut intake = EventIntake::new(store, None, true);
520        let ev = sample_event("00000000-0000-0000-0000-000000000001", "task_assigned");
521        intake.ingest_batch(
522            vec![ev],
523            Vec::new(),
524            PaginatedMeta {
525                has_more: false,
526                next_cursor: Some("c_fail".to_string()),
527                total_count: Some(1),
528            },
529        );
530
531        assert!(intake.commit().is_err());
532        assert_eq!(intake.pending_seen.len(), 1);
533        assert!(matches!(intake.pending_cursor, PendingCursor::Update(_)));
534    }
535}