1use std::{
4 collections::{HashMap, VecDeque},
5 fs::File,
6 io::{self, BufRead, BufReader, Read, Seek, SeekFrom},
7 path::{Path, PathBuf},
8 sync::{
9 atomic::{AtomicBool, AtomicU64, Ordering},
10 Arc, Condvar, Mutex,
11 },
12 time::Instant,
13};
14
15use subc_control::{
16 ActivityActor, ActivityCursor, ActivityEntry, ActivityQueryRequest, ActivityQueryResponse,
17 ActivityRecordRequest, ACTIVITY_CURSOR_INCARNATION_MISMATCH, ACTIVITY_CURSOR_TOO_OLD,
18 ACTIVITY_KIND_FORBIDDEN, ACTIVITY_MALFORMED, ACTIVITY_RATE_LIMITED, ACTIVITY_SUBSCRIBER_LAGGED,
19 ACTIVITY_TOO_LARGE, ACTIVITY_UNAVAILABLE,
20};
21use tokio::sync::{mpsc, oneshot};
22
23const QUEUE_CAPACITY: usize = 64;
24const RING_CAPACITY: usize = 1024;
25const SUBSCRIBER_CAPACITY: usize = 256;
26
27#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
29#[serde(default)]
30pub struct ActivityConfig {
31 pub max_age_days: u32,
32 pub max_total_mb: u32,
33}
34
35impl Default for ActivityConfig {
36 fn default() -> Self {
37 Self {
38 max_age_days: 30,
39 max_total_mb: 64,
40 }
41 }
42}
43
44impl ActivityConfig {
45 fn retention(self) -> cortexkit_log::Retention {
46 let total = self.max_total_mb.max(1);
47 let file_mb = total.div_ceil(256);
50 cortexkit_log::Retention {
51 max_file_mb: file_mb,
52 keep: (total / file_mb - 1) as u8,
53 max_age_days: self.max_age_days,
54 }
55 }
56}
57
58#[derive(Default)]
59struct SharedWriter {
60 writer: Mutex<Option<Writer>>,
61 ready: Condvar,
62 failed: AtomicBool,
63}
64
65impl SharedWriter {
66 fn lock(&self) -> Result<std::sync::MutexGuard<'_, Option<Writer>>, &'static str> {
67 let mut writer = self.writer.lock().unwrap_or_else(|p| p.into_inner());
68 while writer.is_none() {
69 if self.failed.load(Ordering::Relaxed) {
70 return Err(ACTIVITY_UNAVAILABLE);
71 }
72 writer = self.ready.wait(writer).unwrap_or_else(|p| p.into_inner());
73 }
74 Ok(writer)
75 }
76}
77
78struct Subscriber {
79 sender: mpsc::Sender<ActivityEntry>,
80 terminal: Arc<Mutex<Option<&'static str>>>,
81}
82
83pub(crate) struct ActivitySubscription {
84 pub(crate) replay: VecDeque<ActivityEntry>,
85 pub(crate) receiver: mpsc::Receiver<ActivityEntry>,
86 terminal: Arc<Mutex<Option<&'static str>>>,
87}
88
89impl ActivitySubscription {
90 pub(crate) fn terminal_code(&self) -> Option<&'static str> {
91 *self.terminal.lock().unwrap_or_else(|p| p.into_inner())
92 }
93}
94
95pub(crate) fn validate(module: &str, request: &ActivityRecordRequest) -> Result<(), &'static str> {
96 let name = request
97 .kind
98 .strip_prefix(module)
99 .and_then(|s| s.strip_prefix('.'));
100 let shortcut = module == "subc"
102 && matches!(
103 request.kind.as_str(),
104 "operator_shortcut" | "operator_shortcut_outcome"
105 );
106 if !shortcut && !name.is_some_and(|name| identifier(name, 64)) {
107 return Err(ACTIVITY_KIND_FORBIDDEN);
108 }
109 let summary_len = request.summary.chars().count();
110 if summary_len == 0 {
111 return Err(ACTIVITY_MALFORMED);
112 }
113 if summary_len > 200 {
114 return Err(ACTIVITY_TOO_LARGE);
115 }
116 if let Some(subject) = &request.subject {
117 if subject.subject_type.len() > 32 || subject.id.chars().count() > 256 {
118 return Err(ACTIVITY_TOO_LARGE);
119 }
120 if !identifier(&subject.subject_type, 32) || subject.id.is_empty() {
121 return Err(ACTIVITY_MALFORMED);
122 }
123 }
124 if let Some(details) = &request.details {
125 let bytes = serde_json::to_vec(details).map_err(|_| ACTIVITY_MALFORMED)?;
126 if bytes.len() > 4096 {
127 return Err(ACTIVITY_TOO_LARGE);
128 }
129 }
130 Ok(())
131}
132
133fn identifier(value: &str, max: usize) -> bool {
134 !value.is_empty()
135 && value.len() <= max
136 && value
137 .bytes()
138 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
139}
140
141struct Append {
142 module: String,
143 request: ActivityRecordRequest,
144 actor: ActivityActor,
145 at_ms: u64,
146 reply: Option<oneshot::Sender<Result<ActivityEntry, &'static str>>>,
147 lifecycle: Option<lifecycle::PendingLifecycle>,
148}
149
150#[derive(Clone)]
152pub(crate) struct ActivityLedger {
153 sender: mpsc::Sender<Append>,
154 read_errors: Arc<AtomicU64>,
155 shared: Arc<SharedWriter>,
156 lifecycle_counters: Arc<lifecycle::LifecycleCounters>,
157}
158
159impl std::fmt::Debug for ActivityLedger {
160 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161 f.debug_struct("ActivityLedger").finish_non_exhaustive()
162 }
163}
164
165impl ActivityLedger {
166 pub(crate) fn read_errors(&self) -> u64 {
167 self.read_errors.load(Ordering::Relaxed)
168 }
169 #[cfg(test)]
170 pub(crate) fn remaining_capacity(&self) -> usize {
171 self.sender.capacity()
172 }
173 #[cfg(test)]
174 pub(crate) fn with_writer_lock(&self, work: impl FnOnce()) {
175 let _guard = self.shared.lock().unwrap();
176 work();
177 }
178 pub(crate) fn open(path: PathBuf, incarnation: String) -> Self {
179 Self::start(path, incarnation, QUEUE_CAPACITY, WriterHooks::default())
180 }
181
182 pub(crate) fn open_config(path: PathBuf, incarnation: String, config: ActivityConfig) -> Self {
183 Self::start_config(
184 path,
185 incarnation,
186 QUEUE_CAPACITY,
187 WriterHooks::default(),
188 config,
189 )
190 }
191
192 pub(crate) fn start(
193 path: PathBuf,
194 incarnation: String,
195 capacity: usize,
196 hooks: WriterHooks,
197 ) -> Self {
198 Self::start_config(
199 path,
200 incarnation,
201 capacity,
202 hooks,
203 ActivityConfig::default(),
204 )
205 }
206
207 fn start_config(
208 path: PathBuf,
209 incarnation: String,
210 capacity: usize,
211 hooks: WriterHooks,
212 config: ActivityConfig,
213 ) -> Self {
214 let (sender, mut receiver) = mpsc::channel::<Append>(capacity);
215 let read_errors = Arc::new(AtomicU64::new(0));
216 let writer_errors = read_errors.clone();
217 let shared = Arc::new(SharedWriter::default());
218 let worker = shared.clone();
219 let started = std::thread::Builder::new()
222 .name("activity-writer".into())
223 .spawn(move || {
224 let writer = Writer::open(path, incarnation, hooks, &writer_errors, config);
225 let pause = writer.hooks.pause_hook();
226 *worker.writer.lock().unwrap_or_else(|p| p.into_inner()) = Some(writer);
227 worker.ready.notify_all();
228 pause.wait();
229 while let Some(append) = receiver.blocking_recv() {
230 let mut append = append;
231 let mut guard = worker.lock().unwrap();
232 let writer = guard.as_mut().unwrap();
233 match writer.prepare_lifecycle(&mut append, &writer_errors) {
234 Ok(false) => {
235 append.lifecycle.as_mut().unwrap().completed = true;
236 continue;
237 }
238 Err(_) => continue,
239 Ok(true) => {}
240 }
241 let result = writer.append(&append);
242 if result.is_ok() {
243 if let Some(pending) = &mut append.lifecycle {
244 pending.completed = true;
245 }
246 }
247 if let Some(reply) = append.reply.take() {
248 let _ = reply.send(result);
249 }
250 }
251 });
252 if let Err(error) = started {
253 tracing::warn!(%error, "activity writer unavailable");
254 let _guard = shared.writer.lock().unwrap_or_else(|p| p.into_inner());
255 shared.failed.store(true, Ordering::Relaxed);
256 shared.ready.notify_all();
257 }
258 Self {
259 sender,
260 read_errors,
261 shared,
262 lifecycle_counters: Arc::default(),
263 }
264 }
265
266 pub(crate) async fn query(
267 &self,
268 request: ActivityQueryRequest,
269 ) -> Result<ActivityQueryResponse, &'static str> {
270 let shared = self.shared.clone();
271 let errors = self.read_errors.clone();
272 tokio::task::spawn_blocking(move || {
273 let pinned = {
274 let guard = shared.lock()?;
275 guard.as_ref().unwrap().capture_read()?
276 };
277 pinned.read(request, &errors)
278 })
279 .await
280 .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
281 }
282
283 pub(crate) async fn subscribe(
284 &self,
285 since: Option<ActivityCursor>,
286 ) -> Result<ActivitySubscription, &'static str> {
287 let shared = self.shared.clone();
288 tokio::task::spawn_blocking(move || shared.lock()?.as_mut().unwrap().subscribe(since))
289 .await
290 .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
291 }
292
293 pub(crate) async fn configure(&self, config: ActivityConfig) -> Result<(), &'static str> {
294 let shared = self.shared.clone();
295 tokio::task::spawn_blocking(move || shared.lock()?.as_mut().unwrap().configure(config))
296 .await
297 .unwrap_or(Err(ACTIVITY_UNAVAILABLE))
298 }
299
300 pub(crate) async fn record(
301 &self,
302 module: String,
303 request: ActivityRecordRequest,
304 actor: ActivityActor,
305 at_ms: u64,
306 ) -> Result<ActivityEntry, &'static str> {
307 let (reply, received) = oneshot::channel();
308 self.sender
309 .try_send(Append {
310 module,
311 request,
312 actor,
313 at_ms,
314 reply: Some(reply),
315 lifecycle: None,
316 })
317 .map_err(|_| ACTIVITY_UNAVAILABLE)?;
318 received.await.unwrap_or(Err(ACTIVITY_UNAVAILABLE))
319 }
320}
321
322#[derive(Default)]
323pub(crate) struct WriterHooks {
324 #[cfg(test)]
325 pub(crate) open_failure: bool,
326 #[cfg(test)]
327 pub(crate) write_failure: bool,
328 #[cfg(test)]
329 pub(crate) pause: Option<Arc<(std::sync::Mutex<bool>, std::sync::Condvar)>>,
330 #[cfg(test)]
331 pub(crate) read_pause: Option<Arc<(std::sync::Barrier, std::sync::Barrier)>>,
332 pub(crate) clock: Option<Arc<dyn Fn() -> u64 + Send + Sync>>,
333}
334
335impl WriterHooks {
336 fn pause_hook(&self) -> Self {
337 Self {
338 #[cfg(test)]
339 pause: self.pause.clone(),
340 ..Default::default()
341 }
342 }
343 fn wait(&self) {
344 #[cfg(test)]
345 if let Some(pause) = &self.pause {
346 let (lock, ready) = &**pause;
347 let mut paused = lock.lock().unwrap();
348 while *paused {
349 paused = ready.wait(paused).unwrap();
350 }
351 }
352 }
353}
354
355struct Bucket {
356 credit_ms: u64,
358 updated_ms: u64,
359}
360
361impl Bucket {
362 fn available(&mut self, now: u64) -> bool {
363 self.credit_ms = self
364 .credit_ms
365 .saturating_add(now.saturating_sub(self.updated_ms))
366 .min(6000);
367 self.updated_ms = self.updated_ms.max(now);
368 self.credit_ms >= 100
369 }
370}
371
372struct Writer {
373 sink: Option<cortexkit_log::LineSink>,
374 path: PathBuf,
375 config: ActivityConfig,
376 next_seq: u64,
377 first_seq: u64,
378 incarnation: String,
379 ring: VecDeque<ActivityEntry>,
380 subscribers: Vec<Subscriber>,
381 buckets: HashMap<String, Bucket>,
382 started: Instant,
383 hooks: WriterHooks,
384}
385
386impl Writer {
387 fn open(
388 path: PathBuf,
389 incarnation: String,
390 hooks: WriterHooks,
391 read_errors: &AtomicU64,
392 config: ActivityConfig,
393 ) -> Self {
394 let opened = (|| -> io::Result<_> {
395 #[cfg(test)]
396 if hooks.open_failure {
397 return Err(io::Error::other("injected open failure"));
398 }
399 let sink = open_sink(&path, config)?;
400 let highest = retained_max(&path, config.retention().keep, read_errors)?;
401 Ok((
402 sink,
403 highest
404 .checked_add(1)
405 .ok_or_else(|| io::Error::other("activity seq exhausted"))?,
406 ))
407 })();
408 let (sink, next_seq) = match opened {
409 Ok((sink, seq)) => (Some(sink), seq),
410 Err(error) => {
411 tracing::warn!(path = %path.display(), %error, "activity ledger unavailable");
412 (None, 1)
413 }
414 };
415 Self {
416 sink,
417 path,
418 config,
419 next_seq,
420 first_seq: next_seq,
421 incarnation,
422 ring: VecDeque::new(),
423 subscribers: Vec::new(),
424 buckets: HashMap::new(),
425 started: Instant::now(),
426 hooks,
427 }
428 }
429
430 fn append(&mut self, append: &Append) -> Result<ActivityEntry, &'static str> {
431 if append.lifecycle.is_none() {
432 validate(&append.module, &append.request)?;
433 }
434 if self.sink.is_none() || self.next_seq == u64::MAX {
435 return Err(ACTIVITY_UNAVAILABLE);
436 }
437 let now = self.hooks.clock.as_ref().map_or_else(
438 || self.started.elapsed().as_millis() as u64,
439 |clock| clock(),
440 );
441 let bucket = self.buckets.entry(append.module.clone()).or_insert(Bucket {
442 credit_ms: 6000,
443 updated_ms: now,
444 });
445 if append.lifecycle.is_none() && !bucket.available(now) {
446 return Err(ACTIVITY_RATE_LIMITED);
447 }
448 let entry = ActivityEntry {
449 seq: self.next_seq,
450 at_ms: append.at_ms,
451 daemon_incarnation: self.incarnation.clone(),
452 module: append.module.clone(),
453 kind: append.request.kind.clone(),
454 summary: append.request.summary.clone(),
455 subject: append.request.subject.clone(),
456 details: append.request.details.clone().unwrap_or_default(),
457 actor: append.actor.clone(),
458 };
459 let written = (|| -> io::Result<()> {
460 #[cfg(test)]
461 if self.hooks.write_failure {
462 return Err(io::Error::other("injected write failure"));
463 }
464 let bytes = serde_json::to_vec(&entry).map_err(io::Error::other)?;
465 self.sink
466 .as_mut()
467 .ok_or_else(|| io::Error::other("activity sink unavailable"))?
468 .write_line(&bytes)
469 })();
470 if let Err(error) = written {
471 self.sink = None;
474 tracing::warn!(kind = %entry.kind, %error, "activity append failed");
475 return Err(ACTIVITY_UNAVAILABLE);
476 }
477 if append.lifecycle.is_none() {
478 bucket.credit_ms -= 100;
479 }
480 self.next_seq += 1;
481 self.ring.push_back(entry.clone());
482 if self.ring.len() > RING_CAPACITY {
483 self.ring.pop_front();
484 }
485 self.subscribers.retain(
486 |subscriber| match subscriber.sender.try_send(entry.clone()) {
487 Ok(()) => true,
488 Err(mpsc::error::TrySendError::Full(_)) => {
489 *subscriber
490 .terminal
491 .lock()
492 .unwrap_or_else(|p| p.into_inner()) = Some(ACTIVITY_SUBSCRIBER_LAGGED);
493 false
494 }
495 Err(mpsc::error::TrySendError::Closed(_)) => false,
496 },
497 );
498 Ok(entry)
499 }
500
501 fn subscribe(
502 &mut self,
503 since: Option<ActivityCursor>,
504 ) -> Result<ActivitySubscription, &'static str> {
505 if let Some(cursor) = &since {
506 if cursor.daemon_incarnation != self.incarnation {
507 return Err(ACTIVITY_CURSOR_INCARNATION_MISMATCH);
508 }
509 if let Some(first) = self.ring.front() {
514 if first.seq > self.first_seq && cursor.seq.saturating_add(1) < first.seq {
515 return Err(ACTIVITY_CURSOR_TOO_OLD);
516 }
517 }
518 }
519 let replay = self
520 .ring
521 .iter()
522 .filter(|entry| since.as_ref().is_some_and(|cursor| entry.seq > cursor.seq))
523 .cloned()
524 .collect();
525 let (sender, receiver) = mpsc::channel(SUBSCRIBER_CAPACITY);
526 let terminal = Arc::new(Mutex::new(None));
527 self.subscribers
528 .retain(|subscriber| !subscriber.sender.is_closed());
529 self.subscribers.push(Subscriber {
530 sender,
531 terminal: terminal.clone(),
532 });
533 Ok(ActivitySubscription {
534 replay,
535 receiver,
536 terminal,
537 })
538 }
539
540 fn configure(&mut self, config: ActivityConfig) -> Result<(), &'static str> {
541 if config == self.config {
542 return Ok(());
543 }
544 let sink = open_sink(&self.path, config).map_err(|error| {
545 tracing::warn!(%error, "activity retention update failed");
546 ACTIVITY_UNAVAILABLE
547 })?;
548 self.sink = Some(sink);
549 self.config = config;
550 Ok(())
551 }
552
553 fn capture_read(&self) -> Result<ActivityRead, &'static str> {
554 if self.sink.is_none() {
555 return Err(ACTIVITY_UNAVAILABLE);
556 }
557 let mut files = Vec::new();
558 for generation in 0..=self.config.retention().keep {
559 match File::open(generation_path(&self.path, generation)) {
560 Ok(file) => {
561 let len = file.metadata().map_err(|_| ACTIVITY_UNAVAILABLE)?.len();
562 files.push((file, len));
563 }
564 Err(error) if error.kind() == io::ErrorKind::NotFound => {}
565 Err(_) => return Err(ACTIVITY_UNAVAILABLE),
566 }
567 }
568 Ok(ActivityRead {
569 files,
570 #[cfg(test)]
571 read_pause: self.hooks.read_pause.clone(),
572 head: ActivityCursor {
573 daemon_incarnation: self.incarnation.clone(),
574 seq: self.next_seq - 1,
575 },
576 })
577 }
578}
579
580struct ActivityRead {
581 files: Vec<(File, u64)>,
582 head: ActivityCursor,
583 #[cfg(test)]
584 read_pause: Option<Arc<(std::sync::Barrier, std::sync::Barrier)>>,
585}
586
587impl ActivityRead {
588 fn read(
589 self,
590 request: ActivityQueryRequest,
591 read_errors: &AtomicU64,
592 ) -> Result<ActivityQueryResponse, &'static str> {
593 #[cfg(test)]
594 if let Some(pause) = &self.read_pause {
595 pause.0.wait();
596 pause.1.wait();
597 }
598 let mut entries = Vec::new();
599 for (file, len) in self.files {
600 for line in BufReader::new(file.take(len)).split(b'\n') {
601 let line = line.map_err(|_| ACTIVITY_UNAVAILABLE)?;
602 let entry: ActivityEntry = match serde_json::from_slice(&line) {
603 Ok(entry) => entry,
604 Err(error) => {
605 read_errors.fetch_add(1, Ordering::Relaxed);
606 tracing::warn!(%error, "skipping unparseable activity line");
607 continue;
608 }
609 };
610 if request
611 .agent
612 .as_ref()
613 .is_some_and(|v| entry.actor.agent_id.as_ref() != Some(v))
614 || request
615 .project
616 .as_ref()
617 .is_some_and(|v| entry.actor.project_root.as_ref() != Some(v))
618 || request.module.as_ref().is_some_and(|v| &entry.module != v)
619 || request.kind.as_ref().is_some_and(|v| &entry.kind != v)
620 || request.since_ms.is_some_and(|v| entry.at_ms < v)
621 || request.until_ms.is_some_and(|v| entry.at_ms > v)
622 || request.cursor.is_some_and(|v| entry.seq >= v)
623 {
624 continue;
625 }
626 entries.push(entry);
627 }
628 }
629 entries.sort_unstable_by_key(|entry| std::cmp::Reverse(entry.seq));
630 entries.truncate(request.limit as usize);
631 Ok(ActivityQueryResponse {
632 cursor: entries.last().map(|entry| entry.seq),
633 head: self.head,
634 entries,
635 })
636 }
637}
638
639fn open_sink(path: &Path, config: ActivityConfig) -> io::Result<cortexkit_log::LineSink> {
640 let mut sink = cortexkit_log::LineSink::open(path, config.retention())?;
641 let mut file = File::open(path)?;
644 if file.metadata()?.len() > 0 {
645 file.seek(SeekFrom::End(-1))?;
646 let mut last = [0];
647 file.read_exact(&mut last)?;
648 if last[0] != b'\n' {
649 sink.write_line(b"")?;
650 }
651 }
652 Ok(sink)
653}
654
655fn generation_path(path: &Path, generation: u8) -> PathBuf {
656 let mut name = path.as_os_str().to_owned();
657 if generation != 0 {
658 name.push(format!(".{generation}"));
659 }
660 PathBuf::from(name)
661}
662
663fn retained_max(path: &Path, keep: u8, read_errors: &AtomicU64) -> io::Result<u64> {
664 let mut highest = 0;
665 for generation in 0..=keep {
666 let file = match File::open(generation_path(path, generation)) {
667 Ok(file) => file,
668 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
669 Err(error) => return Err(error),
670 };
671 for line in BufReader::new(file).split(b'\n') {
672 match serde_json::from_slice::<ActivityEntry>(&line?) {
673 Ok(entry) => highest = highest.max(entry.seq),
674 Err(error) => {
675 read_errors.fetch_add(1, Ordering::Relaxed);
676 tracing::warn!(%error, "skipping unparseable activity line");
677 }
678 }
679 }
680 }
681 Ok(highest)
682}
683
684#[cfg(test)]
685#[path = "activity_tests.rs"]
686mod tests;
687
688#[path = "activity_lifecycle.rs"]
689mod lifecycle;
690
691pub(crate) use lifecycle::{ExecOutcome, ExecOwner, ExecState, ImageFacts, RegistrationImage};
692
693#[cfg(test)]
694#[path = "activity_lifecycle_tests.rs"]
695mod lifecycle_tests;