1use std::sync::Arc;
57use std::time::Duration;
58
59use parking_lot::Mutex;
60use tokio::sync::mpsc::UnboundedSender;
61use tracing::{debug, info, warn};
62
63use crate::error::{Result, TokenError};
64use crate::seed_agent::{
65 SeedAgentEarmarkManager, SeedAgentStatus, MONTH_MILLIS,
66};
67use crate::seed_agent_gossip::SeedAgentGossipMessage;
68use tenzro_types::primitives::Timestamp;
69
70pub const DEFAULT_DAEMON_POLL_INTERVAL_SECS: u64 = 6 * 60 * 60;
74
75pub const DEFAULT_MIN_REFILL_INTERVAL_MS: i64 = MONTH_MILLIS;
79
80pub const DEFAULT_DAEMON_QUARANTINE_GRACE_MS: i64 =
85 crate::seed_agent::DEFAULT_QUARANTINE_GRACE_MS;
86
87pub type TickAuthorityFn = Arc<dyn Fn() -> bool + Send + Sync + 'static>;
90
91#[derive(Clone)]
93pub struct SeedAgentDaemonConfig {
94 pub poll_interval_secs: u64,
96 pub min_refill_interval_ms: i64,
99 pub quarantine_grace_ms: i64,
103}
104
105impl Default for SeedAgentDaemonConfig {
106 fn default() -> Self {
107 Self {
108 poll_interval_secs: DEFAULT_DAEMON_POLL_INTERVAL_SECS,
109 min_refill_interval_ms: DEFAULT_MIN_REFILL_INTERVAL_MS,
110 quarantine_grace_ms: DEFAULT_DAEMON_QUARANTINE_GRACE_MS,
111 }
112 }
113}
114
115#[derive(Debug, Clone, Default, PartialEq, Eq)]
117pub struct TickOutcome {
118 pub paused_count: u32,
121 pub refilled_count: u32,
123 pub total_granted_wei: u128,
125 pub master_switch_skipped: bool,
127 pub authority_skipped: bool,
129 pub quarantined_count: u32,
131 pub terminated_count: u32,
133 pub surplus_disposed_wei: u128,
135}
136
137pub type SurplusDispositionFn = Arc<
144 dyn Fn(&crate::seed_agent::SurplusDisposition) -> Result<()> + Send + Sync + 'static,
145>;
146
147pub struct SeedAgentDaemon {
149 manager: Arc<SeedAgentEarmarkManager>,
150 gossip_tx: Option<UnboundedSender<SeedAgentGossipMessage>>,
151 tick_authority: Option<TickAuthorityFn>,
152 surplus_disposition: Option<SurplusDispositionFn>,
153 config: SeedAgentDaemonConfig,
154 last_outcome: Mutex<Option<TickOutcome>>,
156}
157
158impl SeedAgentDaemon {
159 pub fn new(manager: Arc<SeedAgentEarmarkManager>) -> Self {
162 Self::with_config(manager, SeedAgentDaemonConfig::default())
163 }
164
165 pub fn with_config(
166 manager: Arc<SeedAgentEarmarkManager>,
167 config: SeedAgentDaemonConfig,
168 ) -> Self {
169 Self {
170 manager,
171 gossip_tx: None,
172 tick_authority: None,
173 surplus_disposition: None,
174 config,
175 last_outcome: Mutex::new(None),
176 }
177 }
178
179 pub fn with_surplus_disposition(
185 mut self,
186 cb: SurplusDispositionFn,
187 ) -> Self {
188 self.surplus_disposition = Some(cb);
189 self
190 }
191
192 pub fn with_gossip(
196 mut self,
197 gossip_tx: UnboundedSender<SeedAgentGossipMessage>,
198 ) -> Self {
199 self.gossip_tx = Some(gossip_tx);
200 self
201 }
202
203 pub fn with_tick_authority(mut self, gate: TickAuthorityFn) -> Self {
206 self.tick_authority = Some(gate);
207 self
208 }
209
210 pub fn last_outcome(&self) -> Option<TickOutcome> {
213 self.last_outcome.lock().clone()
214 }
215
216 pub fn spawn(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
220 let poll = Duration::from_secs(self.config.poll_interval_secs);
221 tokio::spawn(async move {
222 let mut ticker = tokio::time::interval(poll);
223 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
224 ticker.tick().await;
227 loop {
228 ticker.tick().await;
229 match self.tick_once(Timestamp::now()) {
230 Ok(outcome) => {
231 if outcome.refilled_count > 0 || outcome.paused_count > 0 {
232 info!(
233 refilled = outcome.refilled_count,
234 paused = outcome.paused_count,
235 granted_wei = %outcome.total_granted_wei,
236 "SeedAgentDaemon tick"
237 );
238 }
239 }
240 Err(e) => {
241 warn!(error = %e, "SeedAgentDaemon: tick_once failed");
242 }
243 }
244 }
245 })
246 }
247
248 pub fn tick_once(&self, now: Timestamp) -> Result<TickOutcome> {
252 let mut outcome = TickOutcome::default();
253
254 let earmark = self.manager.earmark();
256 if !earmark.enabled {
257 debug!("SeedAgentDaemon: earmark disabled, tick is a no-op");
258 outcome.master_switch_skipped = true;
259 *self.last_outcome.lock() = Some(outcome.clone());
260 return Ok(outcome);
261 }
262
263 if let Some(gate) = &self.tick_authority {
265 if !gate() {
266 debug!("SeedAgentDaemon: not tick authority, skipping");
267 outcome.authority_skipped = true;
268 *self.last_outcome.lock() = Some(outcome.clone());
269 return Ok(outcome);
270 }
271 }
272
273 for agent in self.manager.list_agents(None) {
275 if !matches!(agent.status, SeedAgentStatus::Active) {
276 continue;
277 }
278 let Some(charter) = self.manager.get_charter(&agent.charter_id) else {
279 if let Err(e) = self
282 .manager
283 .set_agent_status(&agent.agent_did, SeedAgentStatus::Paused)
284 {
285 warn!(
286 agent = %agent.agent_did,
287 error = %e,
288 "SeedAgentDaemon: failed to pause dangling agent"
289 );
290 continue;
291 }
292 outcome.paused_count = outcome.paused_count.saturating_add(1);
293 self.broadcast_status_change(&agent.agent_did, SeedAgentStatus::Paused);
294 continue;
295 };
296 if !charter.enabled || is_charter_past_sunset(&charter, now) {
297 if let Err(e) = self
298 .manager
299 .set_agent_status(&agent.agent_did, SeedAgentStatus::Paused)
300 {
301 warn!(
302 agent = %agent.agent_did,
303 error = %e,
304 "SeedAgentDaemon: failed to pause sunsetted agent"
305 );
306 continue;
307 }
308 outcome.paused_count = outcome.paused_count.saturating_add(1);
309 self.broadcast_status_change(&agent.agent_did, SeedAgentStatus::Paused);
310 }
311 }
312
313 for agent in self.manager.list_agents(None) {
316 if !matches!(agent.status, SeedAgentStatus::Active) {
317 continue;
318 }
319 let Some(charter) = self.manager.get_charter(&agent.charter_id) else {
320 continue;
321 };
322 if !charter.enabled || is_charter_past_sunset(&charter, now) {
323 continue;
324 }
325 let last_ms = agent.last_active.as_millis();
326 let now_ms = now.as_millis();
327 if now_ms.saturating_sub(last_ms) < self.config.min_refill_interval_ms {
331 continue;
332 }
333
334 let requested = charter.spend_caps.monthly_cap_wei;
337 if requested == 0 {
338 continue;
339 }
340 match self.manager.refill_agent_monthly(&agent.agent_did, requested, now) {
341 Ok(result) if result.granted_wei > 0 => {
342 outcome.refilled_count = outcome.refilled_count.saturating_add(1);
343 outcome.total_granted_wei = outcome
344 .total_granted_wei
345 .saturating_add(result.granted_wei);
346 self.broadcast_refill_completed(
347 &agent.agent_did,
348 result.granted_wei,
349 result.month,
350 );
351 }
352 Ok(_) => {
353 debug!(
357 agent = %agent.agent_did,
358 "SeedAgentDaemon: refill returned 0 granted (sunset/exhausted)"
359 );
360 }
361 Err(e) => {
362 warn!(
363 agent = %agent.agent_did,
364 error = %e,
365 "SeedAgentDaemon: refill failed"
366 );
367 }
368 }
369 }
370
371 match self.manager.sunset_wind_down_sweep(now, self.config.quarantine_grace_ms) {
379 Ok(report) => {
380 outcome.quarantined_count = report.quarantined.len() as u32;
381 outcome.terminated_count = report.terminated.len() as u32;
382 for agent_did in &report.quarantined {
383 self.broadcast_status_change(agent_did, SeedAgentStatus::Quarantined);
384 }
385 for agent_did in &report.terminated {
386 self.broadcast_status_change(agent_did, SeedAgentStatus::Terminated);
387 }
388 if let Some(disposition) = report.surplus {
389 outcome.surplus_disposed_wei = disposition.total_wei;
390 info!(
391 total = %disposition.total_wei,
392 burn = %disposition.burn_wei,
393 treasury = %disposition.treasury_wei,
394 "SeedAgentDaemon: surplus disposed at sunset"
395 );
396 if let Some(tx) = &self.gossip_tx {
399 let earmark = self.manager.earmark();
400 let msg = SeedAgentGossipMessage::EarmarkUpdated(earmark);
401 let _ = tx.send(msg);
402 }
403 if let Some(cb) = &self.surplus_disposition {
405 if let Err(e) = cb(&disposition) {
406 warn!(
407 error = %e,
408 "SeedAgentDaemon: surplus disposition callback failed"
409 );
410 }
411 }
412 }
413 }
414 Err(e) => {
415 warn!(error = %e, "SeedAgentDaemon: sunset_wind_down_sweep failed");
416 }
417 }
418
419 *self.last_outcome.lock() = Some(outcome.clone());
420 Ok(outcome)
421 }
422
423 fn broadcast_refill_completed(
426 &self,
427 agent_did: &str,
428 granted_wei: u128,
429 month: u8,
430 ) {
431 let Some(tx) = &self.gossip_tx else { return };
432 let snapshot = self.manager.earmark();
433 let msg = SeedAgentGossipMessage::MonthlyRefillCompleted {
434 agent_did: agent_did.to_string(),
435 granted_wei,
436 month,
437 earmark_snapshot: snapshot,
438 };
439 if tx.send(msg).is_err() {
440 debug!("SeedAgentDaemon: gossip channel closed, dropping refill broadcast");
441 }
442 }
443
444 fn broadcast_status_change(&self, agent_did: &str, status: SeedAgentStatus) {
445 let Some(tx) = &self.gossip_tx else { return };
446 let msg = SeedAgentGossipMessage::AgentStatusChanged {
447 agent_did: agent_did.to_string(),
448 status,
449 };
450 if tx.send(msg).is_err() {
451 debug!("SeedAgentDaemon: gossip channel closed, dropping status broadcast");
452 }
453 }
454}
455
456fn is_charter_past_sunset(
460 charter: &crate::seed_agent::Charter,
461 now: Timestamp,
462) -> bool {
463 let sunset_ms = charter.sunset.as_millis();
464 sunset_ms != 0 && now.as_millis() >= sunset_ms
465}
466
467impl SeedAgentDaemonConfig {
470 pub fn validate(&self) -> Result<()> {
471 if self.poll_interval_secs == 0 {
472 return Err(TokenError::InvalidParameter(
473 "SeedAgentDaemonConfig.poll_interval_secs == 0".into(),
474 ));
475 }
476 if self.min_refill_interval_ms <= 0 {
477 return Err(TokenError::InvalidParameter(
478 "SeedAgentDaemonConfig.min_refill_interval_ms <= 0".into(),
479 ));
480 }
481 if self.quarantine_grace_ms <= 0 {
482 return Err(TokenError::InvalidParameter(
483 "SeedAgentDaemonConfig.quarantine_grace_ms <= 0".into(),
484 ));
485 }
486 Ok(())
487 }
488}
489
490#[cfg(test)]
491mod tests {
492 use super::*;
493 use crate::seed_agent::{
494 Charter, CounterpartyFilter, DecaySchedule, OperationKind,
495 SeedAgentRecord, SpendCaps, TreasuryEarmark, DEFAULT_SURPLUS_BURN_BPS,
496 };
497 use tenzro_types::primitives::Hash;
498 use tokio::sync::mpsc;
499
500 fn h(byte: u8) -> Hash {
501 Hash::new([byte; 32])
502 }
503
504 fn seeded_manager(now: Timestamp) -> Arc<SeedAgentEarmarkManager> {
505 let mgr = Arc::new(SeedAgentEarmarkManager::new());
506 let earmark = TreasuryEarmark {
507 name: "SeedAgent".into(),
508 initial_allocation_wei: 10_000,
509 allocation_remaining_wei: 10_000,
510 total_drawn_wei: 0,
511 bootstrap_start: now,
512 bootstrap_end: Timestamp::new(now.as_millis() + MONTH_MILLIS * 12),
513 decay_schedule: DecaySchedule::default_with_base(100),
514 seed_agent_count: 0,
515 charter_ids: Vec::new(),
516 enabled: true,
517 surplus_burn_bps: DEFAULT_SURPLUS_BURN_BPS,
518 current_month: 0,
519 month_drawn_wei: 0,
520 };
521 mgr.apply_earmark(earmark).unwrap();
522
523 let charter = Charter {
524 charter_id: h(0xC1),
525 name: "InferenceLoad".into(),
526 purpose: "test".into(),
527 operations: vec![OperationKind::InferenceConsumer],
528 spend_caps: SpendCaps {
529 daily_cap_wei: 50,
530 monthly_cap_wei: 100,
531 per_tx_cap_wei: 10,
532 },
533 target_throughput: None,
534 counterparty_filter: CounterpartyFilter::default(),
535 sunset: Timestamp::new(now.as_millis() + MONTH_MILLIS * 12),
536 enabled: true,
537 };
538 mgr.upsert_charter(charter).unwrap();
539
540 mgr.register_agent(SeedAgentRecord::new(
542 "did:tenzro:machine:seed:a".into(),
543 "did:tenzro:org:treasury:seedagents".into(),
544 h(0xC1),
545 now,
546 ))
547 .unwrap();
548 mgr
549 }
550
551 #[test]
552 fn tick_skips_when_master_switch_off() {
553 let now = Timestamp::new(0);
554 let mgr = seeded_manager(now);
555 let mut earmark = mgr.earmark();
557 earmark.enabled = false;
558 mgr.apply_earmark(earmark).unwrap();
559
560 let daemon = SeedAgentDaemon::new(mgr.clone());
561 let later = Timestamp::new(MONTH_MILLIS + 1);
562 let outcome = daemon.tick_once(later).unwrap();
563 assert!(outcome.master_switch_skipped);
564 assert_eq!(outcome.refilled_count, 0);
565 assert_eq!(mgr.earmark().allocation_remaining_wei, 10_000);
567 }
568
569 #[test]
570 fn tick_skips_when_authority_returns_false() {
571 let now = Timestamp::new(0);
572 let mgr = seeded_manager(now);
573 let daemon = SeedAgentDaemon::new(mgr.clone())
574 .with_tick_authority(Arc::new(|| false));
575 let later = Timestamp::new(MONTH_MILLIS + 1);
576 let outcome = daemon.tick_once(later).unwrap();
577 assert!(outcome.authority_skipped);
578 assert_eq!(outcome.refilled_count, 0);
579 }
580
581 #[test]
582 fn tick_refills_eligible_agent_and_broadcasts() {
583 let now = Timestamp::new(0);
584 let mgr = seeded_manager(now);
585 let (tx, mut rx) = mpsc::unbounded_channel();
586 let daemon = SeedAgentDaemon::new(mgr.clone()).with_gossip(tx);
587
588 let later = Timestamp::new(MONTH_MILLIS + 1);
589 let outcome = daemon.tick_once(later).unwrap();
590 assert_eq!(outcome.refilled_count, 1);
591 assert_eq!(outcome.total_granted_wei, 100);
593
594 let msg = rx.try_recv().expect("missing refill broadcast");
596 match msg {
597 SeedAgentGossipMessage::MonthlyRefillCompleted {
598 granted_wei,
599 month,
600 earmark_snapshot,
601 ..
602 } => {
603 assert_eq!(granted_wei, 100);
604 assert_eq!(month, 1);
607 assert_eq!(earmark_snapshot.allocation_remaining_wei, 9_900);
608 assert_eq!(earmark_snapshot.total_drawn_wei, 100);
609 }
610 other => panic!("unexpected gossip: {:?}", other),
611 }
612 }
613
614 #[test]
615 fn tick_skips_recently_active_agent() {
616 let now = Timestamp::new(0);
617 let mgr = seeded_manager(now);
618 let daemon = SeedAgentDaemon::new(mgr.clone());
619
620 let too_soon = Timestamp::new(MONTH_MILLIS - 1);
623 let outcome = daemon.tick_once(too_soon).unwrap();
624 assert_eq!(outcome.refilled_count, 0);
625 assert_eq!(mgr.earmark().total_drawn_wei, 0);
626 }
627
628 #[test]
629 fn tick_pauses_agent_under_disabled_charter() {
630 let now = Timestamp::new(0);
631 let mgr = seeded_manager(now);
632
633 let mut charter = mgr.get_charter(&h(0xC1)).unwrap();
635 charter.enabled = false;
636 mgr.upsert_charter(charter).unwrap();
637
638 let (tx, mut rx) = mpsc::unbounded_channel();
639 let daemon = SeedAgentDaemon::new(mgr.clone()).with_gossip(tx);
640 let later = Timestamp::new(MONTH_MILLIS + 1);
641 let outcome = daemon.tick_once(later).unwrap();
642 assert_eq!(outcome.paused_count, 1);
643 assert_eq!(outcome.refilled_count, 0);
644 assert_eq!(outcome.quarantined_count, 1);
647
648 let updated = mgr.get_agent("did:tenzro:machine:seed:a").unwrap();
649 assert_eq!(updated.status, SeedAgentStatus::Quarantined);
650
651 let mut saw_paused = false;
654 let mut saw_quarantined = false;
655 while let Ok(msg) = rx.try_recv() {
656 if let SeedAgentGossipMessage::AgentStatusChanged { agent_did, status } = msg {
657 assert_eq!(agent_did, "did:tenzro:machine:seed:a");
658 match status {
659 SeedAgentStatus::Paused => saw_paused = true,
660 SeedAgentStatus::Quarantined => saw_quarantined = true,
661 _ => {}
662 }
663 }
664 }
665 assert!(saw_paused, "missing Paused broadcast");
666 assert!(saw_quarantined, "missing Quarantined broadcast");
667 }
668
669 #[test]
670 fn tick_pauses_agent_past_sunset() {
671 let now = Timestamp::new(0);
672 let mgr = seeded_manager(now);
673
674 let mut charter = mgr.get_charter(&h(0xC1)).unwrap();
676 charter.sunset = Timestamp::new(MONTH_MILLIS);
677 mgr.upsert_charter(charter).unwrap();
678
679 let daemon = SeedAgentDaemon::new(mgr.clone());
680 let later = Timestamp::new(MONTH_MILLIS * 2);
681 let outcome = daemon.tick_once(later).unwrap();
682 assert_eq!(outcome.paused_count, 1);
683 assert_eq!(outcome.quarantined_count, 1);
685
686 let updated = mgr.get_agent("did:tenzro:machine:seed:a").unwrap();
687 assert_eq!(updated.status, SeedAgentStatus::Quarantined);
688 }
689
690 #[test]
691 fn tick_no_op_for_paused_or_terminated_agents() {
692 let now = Timestamp::new(0);
693 let mgr = seeded_manager(now);
694 mgr.set_agent_status("did:tenzro:machine:seed:a", SeedAgentStatus::Paused)
695 .unwrap();
696
697 let daemon = SeedAgentDaemon::new(mgr.clone());
698 let later = Timestamp::new(MONTH_MILLIS * 2);
699 let outcome = daemon.tick_once(later).unwrap();
700 assert_eq!(outcome.refilled_count, 0);
701 assert_eq!(outcome.paused_count, 0);
702 }
703
704 #[test]
705 fn config_validate_rejects_zero_poll() {
706 let cfg = SeedAgentDaemonConfig {
707 poll_interval_secs: 0,
708 min_refill_interval_ms: MONTH_MILLIS,
709 quarantine_grace_ms: DEFAULT_DAEMON_QUARANTINE_GRACE_MS,
710 };
711 assert!(cfg.validate().is_err());
712 }
713
714 #[test]
715 fn config_validate_rejects_non_positive_grace() {
716 let cfg = SeedAgentDaemonConfig {
717 poll_interval_secs: DEFAULT_DAEMON_POLL_INTERVAL_SECS,
718 min_refill_interval_ms: MONTH_MILLIS,
719 quarantine_grace_ms: 0,
720 };
721 assert!(cfg.validate().is_err());
722 }
723
724 #[test]
725 fn tick_runs_sunset_sweep_and_broadcasts_quarantined() {
726 let now = Timestamp::new(0);
727 let mgr = seeded_manager(now);
728
729 mgr.set_agent_status("did:tenzro:machine:seed:a", SeedAgentStatus::Paused)
732 .unwrap();
733 let mut charter = mgr.get_charter(&h(0xC1)).unwrap();
734 charter.enabled = false;
735 mgr.upsert_charter(charter).unwrap();
736
737 let (tx, mut rx) = mpsc::unbounded_channel();
738 let daemon = SeedAgentDaemon::new(mgr.clone()).with_gossip(tx);
739 let later = Timestamp::new(MONTH_MILLIS + 1);
740 let outcome = daemon.tick_once(later).unwrap();
741 assert_eq!(outcome.quarantined_count, 1);
742 assert_eq!(outcome.terminated_count, 0);
743
744 let mut saw_quarantine = false;
746 while let Ok(msg) = rx.try_recv() {
747 if let SeedAgentGossipMessage::AgentStatusChanged { status, .. } = msg {
748 if status == SeedAgentStatus::Quarantined {
749 saw_quarantine = true;
750 break;
751 }
752 }
753 }
754 assert!(saw_quarantine, "expected Quarantined status broadcast");
755 }
756
757 #[test]
758 fn tick_runs_sunset_sweep_and_invokes_surplus_callback() {
759 use std::sync::atomic::{AtomicU64, Ordering};
760
761 let now = Timestamp::new(0);
762 let mgr = seeded_manager(now);
763
764 let mut charter = mgr.get_charter(&h(0xC1)).unwrap();
766 charter.sunset = Timestamp::new(1);
767 mgr.upsert_charter(charter).unwrap();
768 mgr.set_agent_status("did:tenzro:machine:seed:a", SeedAgentStatus::Terminated)
770 .unwrap();
771
772 let burned = Arc::new(AtomicU64::new(0));
773 let burned_cb = burned.clone();
774 let cb: SurplusDispositionFn = Arc::new(move |d| {
775 burned_cb.store(d.burn_wei as u64, Ordering::SeqCst);
776 Ok(())
777 });
778 let daemon = SeedAgentDaemon::new(mgr.clone()).with_surplus_disposition(cb);
779
780 let later = Timestamp::new(MONTH_MILLIS + 1);
781 let outcome = daemon.tick_once(later).unwrap();
782 assert_eq!(outcome.surplus_disposed_wei, 10_000);
784 assert_eq!(burned.load(Ordering::SeqCst), 5_000);
785 assert_eq!(mgr.earmark().allocation_remaining_wei, 0);
787 assert!(!mgr.earmark().enabled);
788 }
789
790 #[test]
791 fn last_outcome_is_stored_after_tick() {
792 let now = Timestamp::new(0);
793 let mgr = seeded_manager(now);
794 let daemon = SeedAgentDaemon::new(mgr.clone());
795 assert!(daemon.last_outcome().is_none());
796
797 let later = Timestamp::new(MONTH_MILLIS + 1);
798 let _ = daemon.tick_once(later).unwrap();
799 let snap = daemon.last_outcome().unwrap();
800 assert_eq!(snap.refilled_count, 1);
801 }
802}