use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use super::*;
pub const DEFAULT_GC_INTERVAL_MS: u64 = 10 * 60 * 1_000;
pub const DEFAULT_GC_RETENTION_DAYS: u64 = 30;
pub const DEFAULT_UNOWNED_LIFETIME_MS: u64 = 24 * 60 * 60 * 1_000;
pub const DEFAULT_IDLE_EXIT_MS: u64 = 30 * 60 * 1_000;
const ORPHAN_GRACE_MS: u64 = 60_000;
const BATCH: usize = 200;
const DEFAULT_MAX_BATCHES: usize = 50;
const SAMPLE: usize = 10;
const COUNT_CAP: usize = 10_000;
const RECLAIM_PAGES: u64 = 16_384;
const DAY_MS: u64 = 24 * 60 * 60 * 1_000;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum UnownedPolicy {
#[default]
Report,
Stop,
Off,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GcConfig {
pub interval_ms: u64,
pub retention_days: u64,
pub unowned: UnownedPolicy,
pub unowned_lifetime_ms: u64,
pub orphan_grace_ms: u64,
pub max_batches: usize,
}
impl Default for GcConfig {
fn default() -> Self {
Self {
interval_ms: DEFAULT_GC_INTERVAL_MS,
retention_days: DEFAULT_GC_RETENTION_DAYS,
unowned: UnownedPolicy::Report,
unowned_lifetime_ms: DEFAULT_UNOWNED_LIFETIME_MS,
orphan_grace_ms: ORPHAN_GRACE_MS,
max_batches: DEFAULT_MAX_BATCHES,
}
}
}
fn env_u64(name: &str, default: u64) -> u64 {
match std::env::var(name) {
Ok(value) => value.trim().parse().unwrap_or_else(|_| {
eprintln!("[apoc] ignoring {name}={value:?}: expected a non-negative integer");
default
}),
Err(_) => default,
}
}
impl GcConfig {
pub fn from_env() -> Self {
let defaults = Self::default();
let unowned = match std::env::var("APOC_GC_UNOWNED") {
Ok(value) => match value.trim() {
"report" => UnownedPolicy::Report,
"stop" => UnownedPolicy::Stop,
"off" => UnownedPolicy::Off,
other => {
eprintln!(
"[apoc] ignoring APOC_GC_UNOWNED={other:?}: expected report, stop, or off"
);
defaults.unowned
}
},
Err(_) => defaults.unowned,
};
Self {
interval_ms: env_u64("APOC_GC_INTERVAL_MS", defaults.interval_ms),
retention_days: env_u64("APOC_GC_RETENTION_DAYS", defaults.retention_days),
orphan_grace_ms: env_u64("APOC_GC_ORPHAN_GRACE_MS", defaults.orphan_grace_ms),
unowned_lifetime_ms: match env_u64(
"APOC_GC_UNOWNED_LIFETIME_MS",
defaults.unowned_lifetime_ms,
) {
0 => defaults.unowned_lifetime_ms,
value => value,
},
unowned,
..defaults
}
}
}
pub fn idle_exit_ms_from_env() -> u64 {
env_u64("APOC_DAEMON_IDLE_EXIT_MS", DEFAULT_IDLE_EXIT_MS)
}
pub fn is_isolated(config: &CoreConfig) -> bool {
let default_state = dirs::data_dir().map(|path| path.join("apoc"));
let default_runtime = std::env::temp_dir().join(format!("apoc-{}", unsafe { libc::geteuid() }));
default_state.as_ref() != Some(&config.state_dir) || config.runtime_dir != default_runtime
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct GcCategory {
pub found: usize,
pub capped: bool,
pub acted: usize,
pub sample: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct GcSpace {
pub auto_vacuum: String,
pub freelist_pages: u64,
pub reclaimed_pages: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct GcReport {
pub applied: bool,
pub started_at: u64,
pub completed_at: u64,
pub retention_days: u64,
pub unowned_policy: UnownedPolicy,
pub orphans: GcCategory,
pub unowned_expired: GcCategory,
pub executions: GcCategory,
pub invocations: GcCategory,
pub process_events: GcCategory,
pub space: GcSpace,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct GcCounts {
pub orphans: usize,
pub unowned_expired: usize,
pub executions: usize,
pub invocations: usize,
pub process_events: usize,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct GcStatus {
pub interval_ms: u64,
pub retention_days: u64,
pub unowned_policy: UnownedPolicy,
pub passes: u64,
pub last_completed_at: Option<u64>,
pub last_error: Option<String>,
pub last: GcCounts,
pub pending: GcCounts,
}
#[derive(Serialize, Deserialize)]
pub(super) struct GcReplay {
fingerprint: String,
report: Option<GcReport>,
}
#[derive(Deserialize)]
pub(super) struct GcParams {
#[serde(default)]
apply: bool,
#[serde(default)]
idempotency_key: Option<String>,
#[serde(default)]
max_batches: Option<usize>,
}
#[derive(Clone)]
pub(super) struct Activity {
connections: Arc<AtomicUsize>,
last_active_at: Arc<AtomicU64>,
}
impl Activity {
pub(super) fn new(at: u64) -> Self {
Self {
connections: Arc::new(AtomicUsize::new(0)),
last_active_at: Arc::new(AtomicU64::new(at)),
}
}
pub(super) fn connection(&self) -> ConnectionGuard {
self.connections.fetch_add(1, Ordering::AcqRel);
self.last_active_at.store(now_ms(), Ordering::Release);
ConnectionGuard(self.clone())
}
}
pub(super) struct ConnectionGuard(Activity);
impl Drop for ConnectionGuard {
fn drop(&mut self) {
self.0.last_active_at.store(now_ms(), Ordering::Release);
self.0.connections.fetch_sub(1, Ordering::AcqRel);
}
}
fn sample<I: IntoIterator<Item = String>>(ids: I) -> Vec<String> {
ids.into_iter().take(SAMPLE).collect()
}
impl Daemon {
pub(crate) async fn collect_garbage(
&self,
apply: bool,
config: &GcConfig,
controller: &ProcessController,
) -> Result<GcReport, String> {
let _pass = self.gc_lock.lock().await;
let started_at = now_ms();
let budget = BATCH.saturating_mul(config.max_batches.max(1));
let mut orphans = self
.executions
.reconcile_orphans(apply, config.orphan_grace_ms, budget)
.await?;
let idle_before = now_ms().saturating_sub(config.orphan_grace_ms);
let code = self
.reconcile_code_orphans(
apply,
idle_before,
budget.saturating_sub(orphans.len()),
controller,
)
.await?;
orphans.extend(code);
let owned = self
.reconcile_owner_orphans(
apply,
idle_before,
budget.saturating_sub(orphans.len()),
controller,
)
.await?;
orphans.extend(owned);
let orphans = GcCategory {
found: orphans.len(),
capped: orphans.len() >= budget,
acted: if apply { orphans.len() } else { 0 },
sample: sample(
orphans
.into_iter()
.map(|(id, cause)| format!("{id} {cause}")),
),
};
let mut unowned_expired = GcCategory::default();
if config.unowned != UnownedPolicy::Off {
let (expired, found, capped) =
self.store.expired_lifetimes(now_ms(), budget, COUNT_CAP)?;
unowned_expired.found = found;
unowned_expired.capped = capped;
unowned_expired.sample = sample(expired.iter().map(|item| item.id.clone()));
if apply && config.unowned == UnownedPolicy::Stop {
for item in expired {
let Some((Some(lifetime_ms), touched_at)) =
self.store.execution_lifetime(&item.id)?
else {
continue;
};
let now = now_ms();
if touched_at.saturating_add(lifetime_ms) > now {
continue;
}
self.polls.notify_cancel(&item.id)?;
let details = serde_json::json!({
"lifetime_ms": lifetime_ms,
"last_touched_at": touched_at,
"expired_at": now
});
match Box::pin(self.executions.expire(
&item.id,
details,
Duration::from_secs(5),
controller,
))
.await
{
Ok(_) => unowned_expired.acted += 1,
Err(error) => {
tracing::warn!(id = %item.id, error = %error, "could not stop an expired execution");
}
}
}
}
}
let mut executions = GcCategory::default();
let mut invocations = GcCategory::default();
let mut process_events = GcCategory::default();
if config.retention_days > 0 {
let cutoff = now_ms().saturating_sub(config.retention_days.saturating_mul(DAY_MS));
(executions.found, executions.capped) =
self.store.count_retention_executions(cutoff, COUNT_CAP)?;
executions.sample = sample(self.store.retention_execution_candidates(cutoff, SAMPLE)?);
(invocations.found, invocations.capped) =
self.store.count_retention_invocations(cutoff, COUNT_CAP)?;
invocations.sample =
sample(self.store.retention_invocation_candidates(cutoff, SAMPLE)?);
(process_events.found, process_events.capped) = self
.store
.count_retention_process_events(cutoff, COUNT_CAP)?;
process_events.sample = sample(
self.store
.retention_process_event_candidates(cutoff, SAMPLE)?,
);
if apply {
let runs = self
.store
.path()
.parent()
.unwrap_or_else(|| Path::new("."))
.join("runs");
for _ in 0..config.max_batches.max(1) {
let ids = self.store.retention_execution_candidates(cutoff, BATCH)?;
if ids.is_empty() {
break;
}
let deleted = self.store.delete_execution_history(&ids)?;
executions.acted += deleted.executions;
for id in &ids {
let directory = runs.join(id);
if directory.exists()
&& let Err(error) = std::fs::remove_dir_all(&directory)
{
tracing::warn!(path = %directory.display(), error = %error, "could not remove retained run logs");
}
}
if deleted.executions == 0 {
break;
}
tokio::task::yield_now().await;
}
for _ in 0..config.max_batches.max(1) {
let ids = self.store.retention_invocation_candidates(cutoff, BATCH)?;
if ids.is_empty() {
break;
}
let deleted = self.store.delete_invocations(&ids)?;
invocations.acted += deleted;
if deleted == 0 {
break;
}
tokio::task::yield_now().await;
}
for _ in 0..config.max_batches.max(1) {
let ids = self
.store
.retention_process_event_candidates(cutoff, BATCH)?;
if ids.is_empty() {
break;
}
let deleted = self.store.delete_process_events(&ids)?;
process_events.acted += deleted;
if deleted == 0 {
break;
}
tokio::task::yield_now().await;
}
}
}
let reclaimed = self
.store
.reclaim_space(if apply { RECLAIM_PAGES } else { 0 })?;
let report = GcReport {
applied: apply,
started_at,
completed_at: now_ms(),
retention_days: config.retention_days,
unowned_policy: config.unowned,
orphans,
unowned_expired,
executions,
invocations,
process_events,
space: GcSpace {
auto_vacuum: reclaimed.auto_vacuum,
freelist_pages: reclaimed.freelist_pages,
reclaimed_pages: reclaimed.reclaimed_pages,
},
};
if apply {
self.record_gc(Ok(&report));
}
Ok(report)
}
async fn reconcile_code_orphans(
&self,
apply: bool,
idle_before: u64,
limit: usize,
controller: &ProcessController,
) -> Result<Vec<(String, &'static str)>, String> {
let mut found = Vec::new();
for record in self.store.recoverable_execution_records()? {
if found.len() >= limit {
break;
}
if !matches!(record.spec.program, ExecutionProgram::Javascript { .. })
|| record.updated_at >= idle_before
{
continue;
}
let state = self.code.read().await.get(record.id.clone()).await.ok();
if let Some(state) = state.filter(|state| code_terminal(state.status)) {
if apply {
self.record_code_execution(&state, controller, None, None, None)?;
}
found.push((record.id, "terminal_program_unsynchronized"));
continue;
}
if self.code.read().await.owns(&record.id).await {
continue;
}
let id = record.id.clone();
if apply {
self.close_code_orphan(record, controller).await?;
}
found.push((id, "code_actor_lost"));
}
Ok(found)
}
async fn close_code_orphan(
&self,
mut record: crate::model::Execution,
controller: &ProcessController,
) -> Result<(), String> {
let details = serde_json::json!({ "cause": "code_actor_lost", "owner_id": record.id });
for child in self.store.execution_children(&record.id)? {
if child.spec.owner_kind != LeaseOwnerKind::Execution
|| matches!(
child.status,
ExecutionStatus::Completed
| ExecutionStatus::Failed
| ExecutionStatus::Canceled
)
{
continue;
}
self.polls.notify_cancel(&child.id)?;
Box::pin(self.executions.cancel_with_cause(
&child.id,
"EXECUTION_OWNER_GONE",
"execution.owner_gone",
details.clone(),
Duration::from_secs(5),
controller,
))
.await?;
}
self.store.interrupt_code_execution(
&record.id,
"INTERRUPTED: no Code Mode actor owns this program any longer",
)?;
let now = now_ms();
record.status = ExecutionStatus::Failed;
record.updated_at = now;
for pass in &mut record.passes {
if pass.completed_at.is_none() {
pass.completed_at = Some(now);
pass.status = ExecutionStatus::Failed;
pass.error_code = Some("EXECUTION_INTERRUPTED".to_string());
}
}
record.result = Some(serde_json::json!({
"error_code": "EXECUTION_INTERRUPTED",
"error": "No Code Mode actor owns this program any longer, so it cannot finish",
"cause": "code_actor_lost"
}));
self.store.put_execution_record(&record)?;
self.store
.release_lease(&format!("execution/{}", record.id))?;
self.lifecycle_event(
"execution.failed",
&format!("execution/{}", record.id),
controller,
Some(record.id.clone()),
None,
record.result.clone().unwrap_or_default(),
)?;
Ok(())
}
async fn reconcile_owner_orphans(
&self,
apply: bool,
idle_before: u64,
limit: usize,
controller: &ProcessController,
) -> Result<Vec<(String, &'static str)>, String> {
let found = self.executions.owner_orphans(idle_before, limit)?;
if apply {
for (id, cause) in &found {
self.polls.notify_cancel(id)?;
Box::pin(self.executions.cancel_with_cause(
id,
"EXECUTION_OWNER_GONE",
"execution.owner_gone",
serde_json::json!({ "cause": cause }),
Duration::from_secs(5),
controller,
))
.await?;
}
}
Ok(found)
}
fn record_gc(&self, outcome: Result<&GcReport, &str>) {
let Ok(mut status) = self.gc_status.lock() else {
return;
};
status.passes += 1;
status.last_completed_at = Some(now_ms());
match outcome {
Ok(report) => {
let counts = |pick: fn(&GcCategory) -> usize| GcCounts {
orphans: pick(&report.orphans),
unowned_expired: pick(&report.unowned_expired),
executions: pick(&report.executions),
invocations: pick(&report.invocations),
process_events: pick(&report.process_events),
};
status.last_error = None;
status.last = counts(|category| category.acted);
status.pending = counts(|category| category.found.saturating_sub(category.acted));
}
Err(error) => status.last_error = Some(error.to_string()),
}
}
pub(super) fn gc_status(&self) -> Option<GcStatus> {
self.gc_status.lock().ok().map(|status| status.clone())
}
pub(super) async fn gc_command(
&self,
params: Value,
controller: &ProcessController,
) -> Result<GcReport, String> {
let params = parse::<GcParams>(params)?;
let mut config = self.gc_config.clone();
if let Some(max_batches) = params.max_batches {
if max_batches == 0 {
return Err("GC_INVALID_LIMIT: max_batches must be at least 1".to_string());
}
config.max_batches = max_batches;
}
if !params.apply {
return self.collect_garbage(false, &config, controller).await;
}
let key = params
.idempotency_key
.filter(|key| !key.trim().is_empty())
.ok_or_else(|| {
"GC_IDEMPOTENCY_KEY_REQUIRED: gc --apply requires an idempotency key".to_string()
})?;
let fingerprint = format!(
"sha256:{:x}",
Sha256::digest(
serde_json::to_vec(&("gc", config.max_batches))
.map_err(|error| error.to_string())?
)
);
let receipt = format!("gc:{key}");
let intent = GcReplay {
fingerprint: fingerprint.clone(),
report: None,
};
if let Some(replay) = self.store.claim_receipt(&receipt, "gc", &intent)? {
if replay.fingerprint != fingerprint {
return Err(format!(
"GC_IDEMPOTENCY_CONFLICT: key \"{key}\" was already used for a different request"
));
}
if let Some(report) = replay.report {
return Ok(report);
}
}
let report = self.collect_garbage(true, &config, controller).await;
let report = match report {
Ok(report) => report,
Err(error) => {
self.record_gc(Err(&error));
return Err(error);
}
};
self.store.put_receipt(
&receipt,
"gc",
&GcReplay {
fingerprint,
report: Some(report.clone()),
},
)?;
Ok(report)
}
pub(super) fn touch_execution(&self, id: &str) {
if let Err(error) = self.store.touch_execution(id, now_ms()) {
tracing::warn!(id = %id, error = %error, "could not record an execution touch");
}
}
pub(super) fn register_unowned_lifetime(
&self,
execution: &crate::model::Execution,
keep_alive: bool,
lifetime_ms: Option<u64>,
) -> Result<(), String> {
if execution.parent_id.is_some() || execution.session_id.is_some() {
return Ok(());
}
let lifetime =
(!keep_alive).then(|| lifetime_ms.unwrap_or(self.gc_config.unowned_lifetime_ms));
self.store
.register_execution_lifetime(&execution.id, lifetime, execution.created_at)
}
pub(crate) fn spawn_gc(daemon: &Arc<Daemon>) -> Option<tokio::task::JoinHandle<()>> {
let period = daemon.gc_config.interval_ms;
if period == 0 {
return None;
}
let daemon = Arc::clone(daemon);
Some(tokio::spawn(async move {
let period = Duration::from_millis(period);
let mut ticks = tokio::time::interval_at(tokio::time::Instant::now() + period, period);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let controller = default_controller();
loop {
tokio::select! {
_ = daemon.shutdown.cancelled() => break,
_ = ticks.tick() => {
let config = daemon.gc_config.clone();
if let Err(error) = daemon.collect_garbage(true, &config, &controller).await {
tracing::warn!(error = %error, "cleanup pass failed");
daemon.record_gc(Err(&error));
}
}
}
}
}))
}
pub(crate) fn spawn_idle_exit(daemon: &Arc<Daemon>) -> Option<tokio::task::JoinHandle<()>> {
if !daemon.isolated {
return None;
}
let idle_ms = daemon.idle_exit_ms;
let daemon = Arc::clone(daemon);
let tick = Duration::from_millis(if idle_ms == 0 {
500
} else {
(idle_ms / 4).clamp(50, 1_000)
});
Some(tokio::spawn(async move {
loop {
tokio::select! {
_ = daemon.shutdown.cancelled() => break,
_ = tokio::time::sleep(tick) => {}
}
if !daemon.state_dir.exists() {
eprintln!(
"[apoc] state directory {} was removed; exiting",
daemon.state_dir.display()
);
daemon.shutdown.cancel();
break;
}
if idle_ms == 0 {
continue;
}
let now = now_ms();
if daemon.busy() {
daemon.activity.last_active_at.store(now, Ordering::Release);
continue;
}
let idle_for =
now.saturating_sub(daemon.activity.last_active_at.load(Ordering::Acquire));
if idle_for >= idle_ms {
eprintln!("[apoc] isolated daemon idle for {idle_for} ms; exiting");
daemon.shutdown.cancel();
break;
}
}
}))
}
fn busy(&self) -> bool {
self.activity.connections.load(Ordering::Acquire) > 0
|| !matches!(self.store.active_execution_count(), Ok(0))
|| !matches!(self.store.running_code_execution_count(), Ok(0))
|| !matches!(self.store.running_hook_delivery_count(), Ok(0))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_a_non_default_state_or_socket_is_isolated() {
let default_state = dirs::data_dir().unwrap().join("apoc");
let default_runtime =
std::env::temp_dir().join(format!("apoc-{}", unsafe { libc::geteuid() }));
let default = CoreConfig {
state_dir: default_state.clone(),
runtime_dir: default_runtime.clone(),
};
assert!(!is_isolated(&default));
assert!(is_isolated(&CoreConfig {
state_dir: default_state.join("elsewhere"),
runtime_dir: default_runtime.clone(),
}));
assert!(is_isolated(&CoreConfig {
state_dir: default_state,
runtime_dir: default_runtime.join("elsewhere"),
}));
}
}