Skip to main content

subc_daemon/
activity.rs

1//! A bounded, single-writer activity ledger. File work never runs on a reader.
2
3use std::{
4    collections::{HashMap, VecDeque},
5    fs::File,
6    io::{self, BufRead, BufReader, Read, Seek, SeekFrom},
7    path::{Path, PathBuf},
8    sync::{
9        atomic::{AtomicBool, AtomicU64, Ordering},
10        Arc, Condvar, Mutex,
11    },
12    time::Instant,
13};
14
15use subc_control::{
16    ActivityActor, ActivityCursor, ActivityEntry, ActivityQueryRequest, ActivityQueryResponse,
17    ActivityRecordRequest, ACTIVITY_CURSOR_INCARNATION_MISMATCH, ACTIVITY_CURSOR_TOO_OLD,
18    ACTIVITY_KIND_FORBIDDEN, ACTIVITY_MALFORMED, ACTIVITY_RATE_LIMITED, ACTIVITY_SUBSCRIBER_LAGGED,
19    ACTIVITY_TOO_LARGE, ACTIVITY_UNAVAILABLE,
20};
21use tokio::sync::{mpsc, oneshot};
22
23const QUEUE_CAPACITY: usize = 64;
24const RING_CAPACITY: usize = 1024;
25const SUBSCRIBER_CAPACITY: usize = 256;
26
27/// Bounds include the active file, not just rotated generations.
28#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
29#[serde(default)]
30pub struct ActivityConfig {
31    pub max_age_days: u32,
32    pub max_total_mb: u32,
33}
34
35impl Default for ActivityConfig {
36    fn default() -> Self {
37        Self {
38            max_age_days: 30,
39            max_total_mb: 64,
40        }
41    }
42}
43
44impl ActivityConfig {
45    fn retention(self) -> cortexkit_log::Retention {
46        let total = self.max_total_mb.max(1);
47        // Retention allows at most 255 rotated files. Scale the file bound for
48        // larger budgets while keeping active + rotated capacity within total.
49        let file_mb = total.div_ceil(256);
50        cortexkit_log::Retention {
51            max_file_mb: file_mb,
52            keep: (total / file_mb - 1) as u8,
53            max_age_days: self.max_age_days,
54        }
55    }
56}
57
58#[derive(Default)]
59struct SharedWriter {
60    writer: Mutex<Option<Writer>>,
61    ready: Condvar,
62    failed: AtomicBool,
63}
64
65impl SharedWriter {
66    fn lock(&self) -> Result<std::sync::MutexGuard<'_, Option<Writer>>, &'static str> {
67        let mut writer = self.writer.lock().unwrap_or_else(|p| p.into_inner());
68        while writer.is_none() {
69            if self.failed.load(Ordering::Relaxed) {
70                return Err(ACTIVITY_UNAVAILABLE);
71            }
72            writer = self.ready.wait(writer).unwrap_or_else(|p| p.into_inner());
73        }
74        Ok(writer)
75    }
76}
77
78struct Subscriber {
79    sender: mpsc::Sender<ActivityEntry>,
80    terminal: Arc<Mutex<Option<&'static str>>>,
81}
82
83pub(crate) struct ActivitySubscription {
84    pub(crate) replay: VecDeque<ActivityEntry>,
85    pub(crate) receiver: mpsc::Receiver<ActivityEntry>,
86    terminal: Arc<Mutex<Option<&'static str>>>,
87}
88
89impl ActivitySubscription {
90    pub(crate) fn terminal_code(&self) -> Option<&'static str> {
91        *self.terminal.lock().unwrap_or_else(|p| p.into_inner())
92    }
93}
94
95pub(crate) fn validate(module: &str, request: &ActivityRecordRequest) -> Result<(), &'static str> {
96    let name = request
97        .kind
98        .strip_prefix(module)
99        .and_then(|s| s.strip_prefix('.'));
100    // These daemon-owned audit kinds are not module-authored activity names.
101    let shortcut = module == "subc"
102        && matches!(
103            request.kind.as_str(),
104            "operator_shortcut" | "operator_shortcut_outcome"
105        );
106    if !shortcut && !name.is_some_and(|name| identifier(name, 64)) {
107        return Err(ACTIVITY_KIND_FORBIDDEN);
108    }
109    let summary_len = request.summary.chars().count();
110    if summary_len == 0 {
111        return Err(ACTIVITY_MALFORMED);
112    }
113    if summary_len > 200 {
114        return Err(ACTIVITY_TOO_LARGE);
115    }
116    if let Some(subject) = &request.subject {
117        if subject.subject_type.len() > 32 || subject.id.chars().count() > 256 {
118            return Err(ACTIVITY_TOO_LARGE);
119        }
120        if !identifier(&subject.subject_type, 32) || subject.id.is_empty() {
121            return Err(ACTIVITY_MALFORMED);
122        }
123    }
124    if let Some(details) = &request.details {
125        let bytes = serde_json::to_vec(details).map_err(|_| ACTIVITY_MALFORMED)?;
126        if bytes.len() > 4096 {
127            return Err(ACTIVITY_TOO_LARGE);
128        }
129    }
130    Ok(())
131}
132
133fn identifier(value: &str, max: usize) -> bool {
134    !value.is_empty()
135        && value.len() <= max
136        && value
137            .bytes()
138            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
139}
140
141struct Append {
142    module: String,
143    request: ActivityRecordRequest,
144    actor: ActivityActor,
145    at_ms: u64,
146    reply: Option<oneshot::Sender<Result<ActivityEntry, &'static str>>>,
147    lifecycle: Option<lifecycle::PendingLifecycle>,
148}
149
150/// Clones share the bounded queue and per-module rate credit until this daemon stops.
151#[derive(Clone)]
152pub(crate) struct ActivityLedger {
153    sender: mpsc::Sender<Append>,
154    read_errors: Arc<AtomicU64>,
155    shared: Arc<SharedWriter>,
156    lifecycle_counters: Arc<lifecycle::LifecycleCounters>,
157}
158
159impl std::fmt::Debug for ActivityLedger {
160    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161        f.debug_struct("ActivityLedger").finish_non_exhaustive()
162    }
163}
164
165impl ActivityLedger {
166    pub(crate) fn read_errors(&self) -> u64 {
167        self.read_errors.load(Ordering::Relaxed)
168    }
169    #[cfg(test)]
170    pub(crate) fn remaining_capacity(&self) -> usize {
171        self.sender.capacity()
172    }
173    #[cfg(test)]
174    pub(crate) fn with_writer_lock(&self, work: impl FnOnce()) {
175        let _guard = self.shared.lock().unwrap();
176        work();
177    }
178    pub(crate) fn open(path: PathBuf, incarnation: String) -> Self {
179        Self::start(path, incarnation, QUEUE_CAPACITY, WriterHooks::default())
180    }
181
182    pub(crate) fn open_config(path: PathBuf, incarnation: String, config: ActivityConfig) -> Self {
183        Self::start_config(
184            path,
185            incarnation,
186            QUEUE_CAPACITY,
187            WriterHooks::default(),
188            config,
189        )
190    }
191
192    pub(crate) fn start(
193        path: PathBuf,
194        incarnation: String,
195        capacity: usize,
196        hooks: WriterHooks,
197    ) -> Self {
198        Self::start_config(
199            path,
200            incarnation,
201            capacity,
202            hooks,
203            ActivityConfig::default(),
204        )
205    }
206
207    fn start_config(
208        path: PathBuf,
209        incarnation: String,
210        capacity: usize,
211        hooks: WriterHooks,
212        config: ActivityConfig,
213    ) -> Self {
214        let (sender, mut receiver) = mpsc::channel::<Append>(capacity);
215        let read_errors = Arc::new(AtomicU64::new(0));
216        let writer_errors = read_errors.clone();
217        let shared = Arc::new(SharedWriter::default());
218        let worker = shared.clone();
219        // A dedicated thread also keeps filesystem stalls off Tokio's workers.
220        // Dropping the last sender ends it; shutdown never joins a stalled sink.
221        let started = std::thread::Builder::new()
222            .name("activity-writer".into())
223            .spawn(move || {
224                let writer = Writer::open(path, incarnation, hooks, &writer_errors, config);
225                let pause = writer.hooks.pause_hook();
226                *worker.writer.lock().unwrap_or_else(|p| p.into_inner()) = Some(writer);
227                worker.ready.notify_all();
228                pause.wait();
229                while let Some(append) = receiver.blocking_recv() {
230                    let mut append = append;
231                    let mut guard = worker.lock().unwrap();
232                    let writer = guard.as_mut().unwrap();
233                    match writer.prepare_lifecycle(&mut append, &writer_errors) {
234                        Ok(false) => {
235                            append.lifecycle.as_mut().unwrap().completed = true;
236                            continue;
237                        }
238                        Err(_) => continue,
239                        Ok(true) => {}
240                    }
241                    let result = writer.append(&append);
242                    if result.is_ok() {
243                        if let Some(pending) = &mut append.lifecycle {
244                            pending.completed = true;
245                        }
246                    }
247                    if let Some(reply) = append.reply.take() {
248                        let _ = reply.send(result);
249                    }
250                }
251            });
252        if let Err(error) = started {
253            tracing::warn!(%error, "activity writer unavailable");
254            let _guard = shared.writer.lock().unwrap_or_else(|p| p.into_inner());
255            shared.failed.store(true, Ordering::Relaxed);
256            shared.ready.notify_all();
257        }
258        Self {
259            sender,
260            read_errors,
261            shared,
262            lifecycle_counters: Arc::default(),
263        }
264    }
265
266    pub(crate) async fn query(
267        &self,
268        request: ActivityQueryRequest,
269    ) -> Result<ActivityQueryResponse, &'static str> {
270        let shared = self.shared.clone();
271        let errors = self.read_errors.clone();
272        tokio::task::spawn_blocking(move || {
273            let pinned = {
274                let guard = shared.lock()?;
275                guard.as_ref().unwrap().capture_read()?
276            };
277            pinned.read(request, &errors)
278        })
279        .await
280        .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
281    }
282
283    pub(crate) async fn subscribe(
284        &self,
285        since: Option<ActivityCursor>,
286    ) -> Result<ActivitySubscription, &'static str> {
287        let shared = self.shared.clone();
288        tokio::task::spawn_blocking(move || shared.lock()?.as_mut().unwrap().subscribe(since))
289            .await
290            .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
291    }
292
293    pub(crate) async fn configure(&self, config: ActivityConfig) -> Result<(), &'static str> {
294        let shared = self.shared.clone();
295        tokio::task::spawn_blocking(move || shared.lock()?.as_mut().unwrap().configure(config))
296            .await
297            .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
298    }
299
300    pub(crate) async fn record(
301        &self,
302        module: String,
303        request: ActivityRecordRequest,
304        actor: ActivityActor,
305        at_ms: u64,
306    ) -> Result<ActivityEntry, &'static str> {
307        let (reply, received) = oneshot::channel();
308        self.sender
309            .try_send(Append {
310                module,
311                request,
312                actor,
313                at_ms,
314                reply: Some(reply),
315                lifecycle: None,
316            })
317            .map_err(|_| ACTIVITY_UNAVAILABLE)?;
318        received.await.unwrap_or(Err(ACTIVITY_UNAVAILABLE))
319    }
320}
321
322#[derive(Default)]
323pub(crate) struct WriterHooks {
324    #[cfg(test)]
325    pub(crate) open_failure: bool,
326    #[cfg(test)]
327    pub(crate) write_failure: bool,
328    #[cfg(test)]
329    pub(crate) pause: Option<Arc<(std::sync::Mutex<bool>, std::sync::Condvar)>>,
330    #[cfg(test)]
331    pub(crate) read_pause: Option<Arc<(std::sync::Barrier, std::sync::Barrier)>>,
332    pub(crate) clock: Option<Arc<dyn Fn() -> u64 + Send + Sync>>,
333}
334
335impl WriterHooks {
336    fn pause_hook(&self) -> Self {
337        Self {
338            #[cfg(test)]
339            pause: self.pause.clone(),
340            ..Default::default()
341        }
342    }
343    fn wait(&self) {
344        #[cfg(test)]
345        if let Some(pause) = &self.pause {
346            let (lock, ready) = &**pause;
347            let mut paused = lock.lock().unwrap();
348            while *paused {
349                paused = ready.wait(paused).unwrap();
350            }
351        }
352    }
353}
354
355struct Bucket {
356    // One token is 100 ms of credit, avoiding floating-point boundary drift.
357    credit_ms: u64,
358    updated_ms: u64,
359}
360
361impl Bucket {
362    fn available(&mut self, now: u64) -> bool {
363        self.credit_ms = self
364            .credit_ms
365            .saturating_add(now.saturating_sub(self.updated_ms))
366            .min(6000);
367        self.updated_ms = self.updated_ms.max(now);
368        self.credit_ms >= 100
369    }
370}
371
372struct Writer {
373    sink: Option<cortexkit_log::LineSink>,
374    path: PathBuf,
375    config: ActivityConfig,
376    next_seq: u64,
377    first_seq: u64,
378    incarnation: String,
379    ring: VecDeque<ActivityEntry>,
380    subscribers: Vec<Subscriber>,
381    buckets: HashMap<String, Bucket>,
382    started: Instant,
383    hooks: WriterHooks,
384}
385
386impl Writer {
387    fn open(
388        path: PathBuf,
389        incarnation: String,
390        hooks: WriterHooks,
391        read_errors: &AtomicU64,
392        config: ActivityConfig,
393    ) -> Self {
394        let opened = (|| -> io::Result<_> {
395            #[cfg(test)]
396            if hooks.open_failure {
397                return Err(io::Error::other("injected open failure"));
398            }
399            let sink = open_sink(&path, config)?;
400            let highest = retained_max(&path, config.retention().keep, read_errors)?;
401            Ok((
402                sink,
403                highest
404                    .checked_add(1)
405                    .ok_or_else(|| io::Error::other("activity seq exhausted"))?,
406            ))
407        })();
408        let (sink, next_seq) = match opened {
409            Ok((sink, seq)) => (Some(sink), seq),
410            Err(error) => {
411                tracing::warn!(path = %path.display(), %error, "activity ledger unavailable");
412                (None, 1)
413            }
414        };
415        Self {
416            sink,
417            path,
418            config,
419            next_seq,
420            first_seq: next_seq,
421            incarnation,
422            ring: VecDeque::new(),
423            subscribers: Vec::new(),
424            buckets: HashMap::new(),
425            started: Instant::now(),
426            hooks,
427        }
428    }
429
430    fn append(&mut self, append: &Append) -> Result<ActivityEntry, &'static str> {
431        if append.lifecycle.is_none() {
432            validate(&append.module, &append.request)?;
433        }
434        if self.sink.is_none() || self.next_seq == u64::MAX {
435            return Err(ACTIVITY_UNAVAILABLE);
436        }
437        let now = self.hooks.clock.as_ref().map_or_else(
438            || self.started.elapsed().as_millis() as u64,
439            |clock| clock(),
440        );
441        let bucket = self.buckets.entry(append.module.clone()).or_insert(Bucket {
442            credit_ms: 6000,
443            updated_ms: now,
444        });
445        if append.lifecycle.is_none() && !bucket.available(now) {
446            return Err(ACTIVITY_RATE_LIMITED);
447        }
448        let entry = ActivityEntry {
449            seq: self.next_seq,
450            at_ms: append.at_ms,
451            daemon_incarnation: self.incarnation.clone(),
452            module: append.module.clone(),
453            kind: append.request.kind.clone(),
454            summary: append.request.summary.clone(),
455            subject: append.request.subject.clone(),
456            details: append.request.details.clone().unwrap_or_default(),
457            actor: append.actor.clone(),
458        };
459        let written = (|| -> io::Result<()> {
460            #[cfg(test)]
461            if self.hooks.write_failure {
462                return Err(io::Error::other("injected write failure"));
463            }
464            let bytes = serde_json::to_vec(&entry).map_err(io::Error::other)?;
465            self.sink
466                .as_mut()
467                .ok_or_else(|| io::Error::other("activity sink unavailable"))?
468                .write_line(&bytes)
469        })();
470        if let Err(error) = written {
471            // A partial write must not be followed by another append until reopen
472            // repairs its boundary. Failed attempts also leave rate credit intact.
473            self.sink = None;
474            tracing::warn!(kind = %entry.kind, %error, "activity append failed");
475            return Err(ACTIVITY_UNAVAILABLE);
476        }
477        if append.lifecycle.is_none() {
478            bucket.credit_ms -= 100;
479        }
480        self.next_seq += 1;
481        self.ring.push_back(entry.clone());
482        if self.ring.len() > RING_CAPACITY {
483            self.ring.pop_front();
484        }
485        self.subscribers.retain(
486            |subscriber| match subscriber.sender.try_send(entry.clone()) {
487                Ok(()) => true,
488                Err(mpsc::error::TrySendError::Full(_)) => {
489                    *subscriber
490                        .terminal
491                        .lock()
492                        .unwrap_or_else(|p| p.into_inner()) = Some(ACTIVITY_SUBSCRIBER_LAGGED);
493                    false
494                }
495                Err(mpsc::error::TrySendError::Closed(_)) => false,
496            },
497        );
498        Ok(entry)
499    }
500
501    fn subscribe(
502        &mut self,
503        since: Option<ActivityCursor>,
504    ) -> Result<ActivitySubscription, &'static str> {
505        if let Some(cursor) = &since {
506            if cursor.daemon_incarnation != self.incarnation {
507                return Err(ACTIVITY_CURSOR_INCARNATION_MISMATCH);
508            }
509            // After restart, a query's head cursor uses the new daemon's ID
510            // with the highest retained seq from the prior daemon. That cursor
511            // remains replayable until ring eviction would leave a gap; then
512            // query recovery is required rather than an incomplete replay.
513            if let Some(first) = self.ring.front() {
514                if first.seq > self.first_seq && cursor.seq.saturating_add(1) < first.seq {
515                    return Err(ACTIVITY_CURSOR_TOO_OLD);
516                }
517            }
518        }
519        let replay = self
520            .ring
521            .iter()
522            .filter(|entry| since.as_ref().is_some_and(|cursor| entry.seq > cursor.seq))
523            .cloned()
524            .collect();
525        let (sender, receiver) = mpsc::channel(SUBSCRIBER_CAPACITY);
526        let terminal = Arc::new(Mutex::new(None));
527        self.subscribers
528            .retain(|subscriber| !subscriber.sender.is_closed());
529        self.subscribers.push(Subscriber {
530            sender,
531            terminal: terminal.clone(),
532        });
533        Ok(ActivitySubscription {
534            replay,
535            receiver,
536            terminal,
537        })
538    }
539
540    fn configure(&mut self, config: ActivityConfig) -> Result<(), &'static str> {
541        if config == self.config {
542            return Ok(());
543        }
544        let sink = open_sink(&self.path, config).map_err(|error| {
545            tracing::warn!(%error, "activity retention update failed");
546            ACTIVITY_UNAVAILABLE
547        })?;
548        self.sink = Some(sink);
549        self.config = config;
550        Ok(())
551    }
552
553    fn capture_read(&self) -> Result<ActivityRead, &'static str> {
554        if self.sink.is_none() {
555            return Err(ACTIVITY_UNAVAILABLE);
556        }
557        let mut files = Vec::new();
558        for generation in 0..=self.config.retention().keep {
559            match File::open(generation_path(&self.path, generation)) {
560                Ok(file) => {
561                    let len = file.metadata().map_err(|_| ACTIVITY_UNAVAILABLE)?.len();
562                    files.push((file, len));
563                }
564                Err(error) if error.kind() == io::ErrorKind::NotFound => {}
565                Err(_) => return Err(ACTIVITY_UNAVAILABLE),
566            }
567        }
568        Ok(ActivityRead {
569            files,
570            #[cfg(test)]
571            read_pause: self.hooks.read_pause.clone(),
572            head: ActivityCursor {
573                daemon_incarnation: self.incarnation.clone(),
574                seq: self.next_seq - 1,
575            },
576        })
577    }
578}
579
580struct ActivityRead {
581    files: Vec<(File, u64)>,
582    head: ActivityCursor,
583    #[cfg(test)]
584    read_pause: Option<Arc<(std::sync::Barrier, std::sync::Barrier)>>,
585}
586
587impl ActivityRead {
588    fn read(
589        self,
590        request: ActivityQueryRequest,
591        read_errors: &AtomicU64,
592    ) -> Result<ActivityQueryResponse, &'static str> {
593        #[cfg(test)]
594        if let Some(pause) = &self.read_pause {
595            pause.0.wait();
596            pause.1.wait();
597        }
598        let mut entries = Vec::new();
599        for (file, len) in self.files {
600            for line in BufReader::new(file.take(len)).split(b'\n') {
601                let line = line.map_err(|_| ACTIVITY_UNAVAILABLE)?;
602                let entry: ActivityEntry = match serde_json::from_slice(&line) {
603                    Ok(entry) => entry,
604                    Err(error) => {
605                        read_errors.fetch_add(1, Ordering::Relaxed);
606                        tracing::warn!(%error, "skipping unparseable activity line");
607                        continue;
608                    }
609                };
610                if request
611                    .agent
612                    .as_ref()
613                    .is_some_and(|v| entry.actor.agent_id.as_ref() != Some(v))
614                    || request
615                        .project
616                        .as_ref()
617                        .is_some_and(|v| entry.actor.project_root.as_ref() != Some(v))
618                    || request.module.as_ref().is_some_and(|v| &entry.module != v)
619                    || request.kind.as_ref().is_some_and(|v| &entry.kind != v)
620                    || request.since_ms.is_some_and(|v| entry.at_ms < v)
621                    || request.until_ms.is_some_and(|v| entry.at_ms > v)
622                    || request.cursor.is_some_and(|v| entry.seq >= v)
623                {
624                    continue;
625                }
626                entries.push(entry);
627            }
628        }
629        entries.sort_unstable_by_key(|entry| std::cmp::Reverse(entry.seq));
630        entries.truncate(request.limit as usize);
631        Ok(ActivityQueryResponse {
632            cursor: entries.last().map(|entry| entry.seq),
633            head: self.head,
634            entries,
635        })
636    }
637}
638
639fn open_sink(path: &Path, config: ActivityConfig) -> io::Result<cortexkit_log::LineSink> {
640    let mut sink = cortexkit_log::LineSink::open(path, config.retention())?;
641    // A killed append without a trailing newline must not join the next valid
642    // record into one unparseable line. Add the missing separator on reopen.
643    let mut file = File::open(path)?;
644    if file.metadata()?.len() > 0 {
645        file.seek(SeekFrom::End(-1))?;
646        let mut last = [0];
647        file.read_exact(&mut last)?;
648        if last[0] != b'\n' {
649            sink.write_line(b"")?;
650        }
651    }
652    Ok(sink)
653}
654
655fn generation_path(path: &Path, generation: u8) -> PathBuf {
656    let mut name = path.as_os_str().to_owned();
657    if generation != 0 {
658        name.push(format!(".{generation}"));
659    }
660    PathBuf::from(name)
661}
662
663fn retained_max(path: &Path, keep: u8, read_errors: &AtomicU64) -> io::Result<u64> {
664    let mut highest = 0;
665    for generation in 0..=keep {
666        let file = match File::open(generation_path(path, generation)) {
667            Ok(file) => file,
668            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
669            Err(error) => return Err(error),
670        };
671        for line in BufReader::new(file).split(b'\n') {
672            match serde_json::from_slice::<ActivityEntry>(&line?) {
673                Ok(entry) => highest = highest.max(entry.seq),
674                Err(error) => {
675                    read_errors.fetch_add(1, Ordering::Relaxed);
676                    tracing::warn!(%error, "skipping unparseable activity line");
677                }
678            }
679        }
680    }
681    Ok(highest)
682}
683
684#[cfg(test)]
685#[path = "activity_tests.rs"]
686mod tests;
687
688#[path = "activity_lifecycle.rs"]
689mod lifecycle;
690
691pub(crate) use lifecycle::{ExecOutcome, ExecOwner, ExecState, ImageFacts, RegistrationImage};
692
693#[cfg(test)]
694#[path = "activity_lifecycle_tests.rs"]
695mod lifecycle_tests;