use super::{RecordStamp, TelemetryRecord};
use crate::runtime::trace::{RequestTrace, TraceSink};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError};
use std::sync::{Arc, Condvar, Mutex as StdMutex};
use std::time::{Duration, Instant};
pub const DEFAULT_QUEUE_CAPACITY: usize = 1024;
pub const DEFAULT_MAX_FILE_BYTES: u64 = 50 * 1024 * 1024;
pub const DEFAULT_MAX_GENERATIONS: u32 = 1;
pub const ROTATE_LOCK_WAIT: Duration = Duration::from_secs(2);
const MAX_GENERATIONS: u32 = agent_core::config::MAX_TELEMETRY_LOG_GENERATIONS;
pub const DEFAULT_SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(2);
fn lock_recover<T>(m: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub fn default_telemetry_log_path() -> Option<PathBuf> {
let home = std::env::var("HOME").ok()?;
Some(PathBuf::from(home).join(".cache/synaps/api-log.jsonl"))
}
pub fn default_trace_log_path() -> Option<PathBuf> {
let home = std::env::var("HOME").ok()?;
Some(PathBuf::from(home).join(".cache/synaps/request-trace.jsonl"))
}
#[derive(Debug, Clone)]
pub struct WriterOptions {
pub telemetry_path: Option<PathBuf>,
pub trace_path: Option<PathBuf>,
pub capacity: usize,
pub max_file_bytes: u64,
pub max_generations: u32,
pub write_delay: Option<Duration>,
pub write_gate: Option<WriteGate>,
}
#[derive(Debug, Clone, Default)]
pub struct WriteGate(Arc<(StdMutex<bool>, Condvar)>);
impl WriteGate {
pub fn closed() -> Self {
Self::default()
}
pub fn open(&self) {
let (open, cv) = &*self.0;
*lock_recover(open) = true;
cv.notify_all();
}
fn wait(&self) {
let (open, cv) = &*self.0;
let mut guard = lock_recover(open);
while !*guard {
guard = cv.wait(guard).unwrap_or_else(|e| e.into_inner());
}
}
}
impl Default for WriterOptions {
fn default() -> Self {
Self {
telemetry_path: None,
trace_path: None,
capacity: DEFAULT_QUEUE_CAPACITY,
max_file_bytes: DEFAULT_MAX_FILE_BYTES,
max_generations: DEFAULT_MAX_GENERATIONS,
write_delay: None,
write_gate: None,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct WriterStats {
pub enqueued: u64,
pub written: u64,
pub dropped: u64,
pub serialization_failures: u64,
pub io_failures: u64,
pub overflow_warnings: u64,
pub serialization_warnings: u64,
pub io_warnings: u64,
pub rotate_warnings: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ShutdownOutcome {
Flushed(WriterStats),
TimedOut(WriterStats),
}
impl ShutdownOutcome {
pub fn stats(&self) -> WriterStats {
match self {
Self::Flushed(s) | Self::TimedOut(s) => *s,
}
}
pub fn is_flushed(&self) -> bool {
matches!(self, Self::Flushed(_))
}
}
#[derive(Debug, Clone)]
pub struct TelemetryWriter {
shared: Arc<Shared>,
stamp: Option<Arc<RecordStamp>>,
}
#[derive(Debug)]
enum Job {
Telemetry(Box<TelemetryRecord>),
Trace(Box<RequestTrace>),
}
#[derive(Debug, Default)]
struct Counters {
enqueued: AtomicU64,
written: AtomicU64,
dropped: AtomicU64,
serialization_failures: AtomicU64,
io_failures: AtomicU64,
overflow_warnings: AtomicU64,
serialization_warnings: AtomicU64,
io_warnings: AtomicU64,
rotate_warnings: AtomicU64,
overflow_warned: AtomicBool,
serialization_warned: AtomicBool,
io_warned: AtomicBool,
rotate_warned: AtomicBool,
}
impl Counters {
fn warn_once(latch: &AtomicBool, count: &AtomicU64, class: &str, msg: &str) {
if !latch.swap(true, Ordering::Relaxed) {
count.fetch_add(1, Ordering::Relaxed);
tracing::warn!(class, "{msg}");
}
}
fn snapshot(&self) -> WriterStats {
WriterStats {
enqueued: self.enqueued.load(Ordering::Relaxed),
written: self.written.load(Ordering::Relaxed),
dropped: self.dropped.load(Ordering::Relaxed),
serialization_failures: self.serialization_failures.load(Ordering::Relaxed),
io_failures: self.io_failures.load(Ordering::Relaxed),
overflow_warnings: self.overflow_warnings.load(Ordering::Relaxed),
serialization_warnings: self.serialization_warnings.load(Ordering::Relaxed),
io_warnings: self.io_warnings.load(Ordering::Relaxed),
rotate_warnings: self.rotate_warnings.load(Ordering::Relaxed),
}
}
}
#[derive(Debug)]
struct Shared {
tx: StdMutex<Option<SyncSender<Job>>>,
latest_traces: StdMutex<std::collections::BTreeMap<String, RequestTrace>>,
worker: StdMutex<Option<std::thread::JoinHandle<()>>>,
done: StdMutex<bool>,
done_cv: Condvar,
counters: Counters,
}
struct WorkerCfg {
telemetry_path: Option<PathBuf>,
trace_path: Option<PathBuf>,
max_file_bytes: u64,
max_generations: u32,
write_delay: Option<Duration>,
write_gate: Option<WriteGate>,
latest_traces: Arc<Shared>,
}
impl TelemetryWriter {
pub fn new(options: WriterOptions) -> Self {
let (tx, rx) = sync_channel::<Job>(options.capacity.max(1));
let shared = Arc::new(Shared {
tx: StdMutex::new(Some(tx)),
latest_traces: StdMutex::new(std::collections::BTreeMap::new()),
worker: StdMutex::new(None),
done: StdMutex::new(false),
done_cv: Condvar::new(),
counters: Counters::default(),
});
let cfg = WorkerCfg {
telemetry_path: options.telemetry_path.or_else(default_telemetry_log_path),
trace_path: options.trace_path.or_else(default_trace_log_path),
max_file_bytes: options.max_file_bytes,
max_generations: options.max_generations.max(1),
write_delay: options.write_delay,
write_gate: options.write_gate,
latest_traces: Arc::clone(&shared),
};
let worker_shared = Arc::clone(&shared);
let handle = std::thread::Builder::new()
.name("synaps-telemetry-writer".to_string())
.spawn(move || worker_loop(rx, cfg, worker_shared))
.expect("spawning the telemetry writer thread cannot fail");
*lock_recover(&shared.worker) = Some(handle);
Self {
shared,
stamp: None,
}
}
pub fn with_stamp(&self, stamp: RecordStamp) -> Self {
Self {
shared: Arc::clone(&self.shared),
stamp: Some(Arc::new(stamp)),
}
}
pub fn with_session(&self, session_id: Option<String>) -> Self {
let mut stamp = self.stamp.as_deref().cloned().unwrap_or_default();
stamp.session_id = session_id;
self.with_stamp(stamp)
}
pub fn enqueue_telemetry(&self, mut record: TelemetryRecord) {
if let Some(stamp) = &self.stamp {
stamp.apply(&mut record);
}
self.enqueue(Job::Telemetry(Box::new(record)));
}
pub fn enqueue_trace(&self, record: RequestTrace) {
lock_recover(&self.shared.latest_traces)
.insert(record.request_id.as_str().to_string(), record.clone());
self.enqueue(Job::Trace(Box::new(record)));
}
pub fn enrich_trace(
&self,
request_id: &crate::runtime::trace::TraceId,
events: Vec<crate::runtime::trace::ToolExecutionEvent>,
) {
if let Some(record) = lock_recover(&self.shared.latest_traces).get_mut(request_id.as_str())
{
record.execution_events = events;
}
}
pub fn trace_snapshot(
&self,
request_id: &crate::runtime::trace::TraceId,
) -> Option<RequestTrace> {
lock_recover(&self.shared.latest_traces)
.get(request_id.as_str())
.cloned()
}
fn enqueue(&self, job: Job) {
let c = &self.shared.counters;
let guard = lock_recover(&self.shared.tx);
let Some(tx) = guard.as_ref() else {
c.dropped.fetch_add(1, Ordering::Relaxed);
return;
};
match tx.try_send(job) {
Ok(()) => {
c.enqueued.fetch_add(1, Ordering::Relaxed);
}
Err(TrySendError::Full(_)) => {
c.dropped.fetch_add(1, Ordering::Relaxed);
Counters::warn_once(
&c.overflow_warned,
&c.overflow_warnings,
"queue_overflow",
"telemetry writer queue full — dropping records (counted, best-effort)",
);
}
Err(TrySendError::Disconnected(_)) => {
c.dropped.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn stats(&self) -> WriterStats {
self.shared.counters.snapshot()
}
pub fn shutdown(&self, timeout: Duration) -> ShutdownOutcome {
drop(lock_recover(&self.shared.tx).take());
let deadline = Instant::now() + timeout;
let mut done = lock_recover(&self.shared.done);
while !*done {
let now = Instant::now();
let Some(remaining) = deadline
.checked_duration_since(now)
.filter(|d| !d.is_zero())
else {
break;
};
let (guard, _timeout) = self
.shared
.done_cv
.wait_timeout(done, remaining)
.unwrap_or_else(std::sync::PoisonError::into_inner);
done = guard;
}
let flushed = *done;
drop(done);
if flushed {
if let Some(handle) = lock_recover(&self.shared.worker).take() {
let _ = handle.join();
}
}
let stats = self.stats();
if flushed {
ShutdownOutcome::Flushed(stats)
} else {
ShutdownOutcome::TimedOut(stats)
}
}
pub async fn shutdown_async(&self, timeout: Duration) -> ShutdownOutcome {
let writer = self.clone();
tokio::task::spawn_blocking(move || writer.shutdown(timeout))
.await
.unwrap_or_else(|_| ShutdownOutcome::TimedOut(self.stats()))
}
}
fn worker_loop(rx: Receiver<Job>, cfg: WorkerCfg, shared: Arc<Shared>) {
while let Ok(job) = rx.recv() {
if let Some(gate) = &cfg.write_gate {
gate.wait();
}
if let Some(delay) = cfg.write_delay {
std::thread::sleep(delay);
}
process_job(job, &cfg, &shared.counters);
}
let mut done = lock_recover(&shared.done);
*done = true;
shared.done_cv.notify_all();
}
fn process_job(job: Job, cfg: &WorkerCfg, c: &Counters) {
let (serialized, path) = match &job {
Job::Telemetry(r) => (serde_json::to_string(r), cfg.telemetry_path.as_deref()),
Job::Trace(r) => {
let latest = lock_recover(&cfg.latest_traces.latest_traces)
.get(r.request_id.as_str())
.cloned()
.unwrap_or_else(|| (**r).clone());
(serde_json::to_string(&latest), cfg.trace_path.as_deref())
}
};
let line = match serialized {
Ok(line) => line,
Err(_) => {
note_serialization_failure(c);
return;
}
};
let Some(path) = path else {
note_io_failure(c);
return;
};
append_private_line(path, &line, cfg.max_file_bytes, cfg.max_generations, c);
}
fn note_serialization_failure(c: &Counters) {
c.serialization_failures.fetch_add(1, Ordering::Relaxed);
Counters::warn_once(
&c.serialization_warned,
&c.serialization_warnings,
"serialization",
"telemetry record failed to serialize — dropped (counted)",
);
}
fn note_io_failure(c: &Counters) {
c.io_failures.fetch_add(1, Ordering::Relaxed);
Counters::warn_once(
&c.io_warned,
&c.io_warnings,
"open_write",
"telemetry log unwritable — records dropped (counted); \
will warn again after the next successful write",
);
}
fn generation_path(path: &std::path::Path, n: u32) -> PathBuf {
let mut p = path.as_os_str().to_owned();
p.push(format!(".{n}"));
PathBuf::from(p)
}
fn append_private_line(
path: &std::path::Path,
line: &str,
max_bytes: u64,
generations: u32,
c: &Counters,
) {
if let Some(parent) = path.parent() {
if agent_core::core::private_fs::ensure_private_dir(parent).is_err() {
note_io_failure(c);
return;
}
}
rotate_if_over(path, max_bytes, generations, c);
match agent_core::core::private_fs::open_private_append(path) {
Ok(mut f) => {
use std::io::Write;
if writeln!(f, "{line}").is_ok() {
c.written.fetch_add(1, Ordering::Relaxed);
c.io_warned.store(false, Ordering::Relaxed);
} else {
note_io_failure(c);
}
}
Err(_) => note_io_failure(c),
}
}
fn over_cap(path: &std::path::Path, max_bytes: u64) -> bool {
std::fs::metadata(path).is_ok_and(|m| m.len() > max_bytes)
}
fn note_rotate_failure(c: &Counters, msg: &str) {
c.io_failures.fetch_add(1, Ordering::Relaxed);
Counters::warn_once(&c.rotate_warned, &c.rotate_warnings, "rotate", msg);
}
fn chmod_private(path: &std::path::Path) {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
}
#[cfg(not(unix))]
let _ = path;
}
fn lock_rotation(path: &std::path::Path) -> Option<std::fs::File> {
use fs4::fs_std::FileExt;
let mut lock_path = path.as_os_str().to_owned();
lock_path.push(".lock");
let file = agent_core::core::private_fs::open_private_append(std::path::Path::new(&lock_path)).ok()?;
let deadline = Instant::now() + ROTATE_LOCK_WAIT;
loop {
match FileExt::try_lock_exclusive(&file) {
Ok(true) => return Some(file),
Ok(false) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(5)),
_ => return None,
}
}
}
fn rotate_if_over(path: &std::path::Path, max_bytes: u64, generations: u32, c: &Counters) {
if !over_cap(path, max_bytes) {
return;
}
let Some(_lock) = lock_rotation(path) else {
note_rotate_failure(c, "telemetry log rotation lock unavailable — appending past the size cap");
return;
};
if !over_cap(path, max_bytes) {
return; }
let n = generations.clamp(1, MAX_GENERATIONS);
for k in (1..n).rev() {
let from = generation_path(path, k);
if std::fs::symlink_metadata(&from).is_err() {
continue;
}
let to = generation_path(path, k + 1);
match std::fs::rename(&from, &to) {
Ok(()) => chmod_private(&to),
Err(_) => note_rotate_failure(c, "telemetry log generation shift failed — a generation may be lost"),
}
}
let rotated = generation_path(path, 1);
if std::fs::rename(path, &rotated).is_ok() {
chmod_private(&rotated);
c.rotate_warned.store(false, Ordering::Relaxed);
} else {
note_rotate_failure(c, "telemetry log rotation failed — appending past the size cap");
}
for k in n + 1..=MAX_GENERATIONS {
let stale = generation_path(path, k);
if std::fs::symlink_metadata(&stale).is_ok() {
let _ = std::fs::remove_file(&stale);
}
}
}
#[derive(Debug)]
pub struct WriterTraceSink {
writer: TelemetryWriter,
}
impl WriterTraceSink {
pub fn new(writer: TelemetryWriter) -> Self {
Self { writer }
}
}
impl TraceSink for WriterTraceSink {
fn emit(&self, record: RequestTrace) {
self.writer.enqueue_trace(record);
}
fn enrich_execution_events(
&self,
request_id: &crate::runtime::trace::TraceId,
events: Vec<crate::runtime::trace::ToolExecutionEvent>,
) {
self.writer.enrich_trace(request_id, events);
}
fn snapshot_for_request(
&self,
request_id: &crate::runtime::trace::TraceId,
) -> Option<RequestTrace> {
self.writer.trace_snapshot(request_id)
}
fn enabled(&self) -> bool {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::telemetry::TelemetryRecord;
use crate::runtime::trace::{
CacheMeta, EndpointMeta, RequestAnatomy, RequestTrace, TimingStages, TraceId,
TraceSchemaVersion, TransportKind, TransportOutcome,
};
use std::time::Instant;
const SENTINEL: &str = "RAW-CONTENT-SENTINEL-9f2c";
fn tmp_writer(
dir: &std::path::Path,
capacity: usize,
delay: Option<Duration>,
) -> (TelemetryWriter, PathBuf, PathBuf) {
let telemetry = dir.join("synaps/api-log.jsonl");
let trace = dir.join("synaps/request-trace.jsonl");
let w = TelemetryWriter::new(WriterOptions {
telemetry_path: Some(telemetry.clone()),
trace_path: Some(trace.clone()),
capacity,
max_file_bytes: DEFAULT_MAX_FILE_BYTES,
max_generations: DEFAULT_MAX_GENERATIONS,
write_delay: delay,
write_gate: None,
});
(w, telemetry, trace)
}
fn sample_telemetry() -> TelemetryRecord {
TelemetryRecord {
ts: 42,
model: "claude-sonnet-4-6".to_string(),
attempt: 1,
total_ms: 100,
..Default::default()
}
}
fn sample_trace() -> RequestTrace {
RequestTrace {
schema: TraceSchemaVersion,
session_id: TraceId::new("sess-w1").unwrap(),
turn_id: TraceId::new("turn-w1").unwrap(),
request_id: TraceId::new("req-w1").unwrap(),
execution_events: Vec::new(),
attempt: 1,
model: agent_core::prompt::QualifiedModelId::parse("anthropic/claude-sonnet-4-6")
.unwrap(),
transport: TransportKind::AnthropicMessages,
endpoint: EndpointMeta::new("api.anthropic.com", "/v1/messages").unwrap(),
anatomy: RequestAnatomy {
system_segment_count: 1,
message_count: 1,
block_count: SENTINEL.len() as u32 % 7,
tool_count: 0,
},
wire: None,
system_segments: vec![],
messages: vec![],
tools: vec![],
cache: CacheMeta::default(),
translation_losses: vec![],
outcome: TransportOutcome {
timings: TimingStages::default(),
retries: vec![],
provider_request_id: None,
http_status: Some(200),
stop_reason: None,
usage: None,
terminal: agent_core::TurnOutcome::Completed,
},
}
}
fn read_lines(path: &std::path::Path) -> Vec<String> {
std::fs::read_to_string(path)
.unwrap_or_default()
.lines()
.map(str::to_string)
.collect()
}
#[test]
fn flush_writes_telemetry_jsonl_in_legacy_format() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, telemetry_path, _) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
let out = w.shutdown(Duration::from_secs(5));
assert!(out.is_flushed());
assert_eq!(out.stats().written, 1);
assert_eq!(out.stats().enqueued, 1);
assert_eq!(out.stats().dropped, 0);
let lines = read_lines(&telemetry_path);
assert_eq!(lines.len(), 1);
let expected = serde_json::to_string(&sample_telemetry()).unwrap();
assert_eq!(lines[0], expected);
assert!(!lines[0].contains("request_id"));
assert!(lines[0].contains("\"model\":\"claude-sonnet-4-6\""));
}
#[test]
fn trace_lines_are_unwrapped_schema_valid_and_content_free() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, trace_path) = tmp_writer(tmp.path(), 16, None);
w.enqueue_trace(sample_trace());
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
let lines = read_lines(&trace_path);
assert_eq!(lines.len(), 1);
let parsed: RequestTrace = serde_json::from_str(&lines[0]).unwrap();
assert_eq!(parsed, sample_trace());
assert!(lines[0].contains("synaps-request-trace/1"));
assert!(!lines[0].contains(SENTINEL));
}
#[test]
fn both_kinds_interleave_without_cross_contamination() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, telemetry_path, trace_path) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
w.enqueue_trace(sample_trace());
w.enqueue_telemetry(sample_telemetry());
let out = w.shutdown(Duration::from_secs(5));
assert_eq!(out.stats().written, 3);
assert_eq!(read_lines(&telemetry_path).len(), 2);
assert_eq!(read_lines(&trace_path).len(), 1);
}
#[test]
fn overflow_increments_dropped_and_warns_once() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 1, Some(Duration::from_millis(200)));
for _ in 0..20 {
w.enqueue_telemetry(sample_telemetry());
}
let stats = w.stats();
assert!(stats.dropped > 0, "queue overflow must drop: {stats:?}");
assert_eq!(
stats.overflow_warnings, 1,
"exactly one overflow warning: {stats:?}"
);
assert_eq!(stats.enqueued + stats.dropped, 20);
w.shutdown(Duration::from_secs(5));
}
#[test]
fn enqueue_after_shutdown_counts_dropped_never_panics() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 16, None);
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
w.enqueue_telemetry(sample_telemetry());
w.enqueue_trace(sample_trace());
assert_eq!(w.stats().dropped, 2);
}
#[test]
fn enqueue_is_fast_even_when_worker_is_slow() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 1, Some(Duration::from_millis(500)));
let start = Instant::now();
for _ in 0..100 {
w.enqueue_telemetry(sample_telemetry());
}
assert!(
start.elapsed() < Duration::from_millis(250),
"enqueue must never block on storage: {:?}",
start.elapsed()
);
let out = w.shutdown(Duration::from_millis(100));
assert!(!out.is_flushed());
}
#[test]
fn trace_sink_emit_is_fast_with_slow_storage() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 1, Some(Duration::from_millis(500)));
let sink = WriterTraceSink::new(w.clone());
assert!(sink.enabled());
let start = Instant::now();
for _ in 0..50 {
sink.emit(sample_trace());
}
assert!(
start.elapsed() < Duration::from_millis(250),
"sink emit must never block on storage: {:?}",
start.elapsed()
);
w.shutdown(Duration::from_millis(100));
}
#[test]
fn broken_path_counts_io_failures_and_warns_once() {
let tmp = tempfile::TempDir::new().unwrap();
let blocker = tmp.path().join("blocked");
std::fs::write(&blocker, b"not a dir").unwrap();
let w = TelemetryWriter::new(WriterOptions {
telemetry_path: Some(blocker.join("api-log.jsonl")),
trace_path: Some(blocker.join("request-trace.jsonl")),
capacity: 16,
max_file_bytes: DEFAULT_MAX_FILE_BYTES,
max_generations: 1,
write_delay: None,
write_gate: None,
});
for _ in 0..3 {
w.enqueue_telemetry(sample_telemetry());
}
let out = w.shutdown(Duration::from_secs(5));
assert!(out.is_flushed(), "broken storage must not stall drain");
let stats = out.stats();
assert_eq!(stats.io_failures, 3);
assert_eq!(stats.io_warnings, 1, "warn once per failure class");
assert_eq!(stats.written, 0);
}
#[test]
fn io_warning_latch_rearms_after_success() {
let tmp = tempfile::TempDir::new().unwrap();
let good = tmp.path().join("synaps/api-log.jsonl");
let blocker = tmp.path().join("blocked");
std::fs::write(&blocker, b"not a dir").unwrap();
let w = TelemetryWriter::new(WriterOptions {
telemetry_path: Some(good.clone()),
trace_path: Some(blocker.join("request-trace.jsonl")),
capacity: 16,
max_file_bytes: DEFAULT_MAX_FILE_BYTES,
max_generations: 1,
write_delay: None,
write_gate: None,
});
w.enqueue_trace(sample_trace()); w.enqueue_telemetry(sample_telemetry()); w.enqueue_trace(sample_trace()); let stats = w.shutdown(Duration::from_secs(5)).stats();
assert_eq!(stats.io_failures, 2);
assert_eq!(stats.io_warnings, 2);
assert_eq!(stats.written, 1);
}
#[test]
fn serialization_failure_counted_and_warned_once() {
let c = Counters::default();
note_serialization_failure(&c);
note_serialization_failure(&c);
let stats = c.snapshot();
assert_eq!(stats.serialization_failures, 2);
assert_eq!(stats.serialization_warnings, 1, "warn once per writer");
}
#[test]
fn rotation_at_cap_preserves_order_and_parseability() {
let tmp = tempfile::TempDir::new().unwrap();
let telemetry_path = tmp.path().join("synaps/api-log.jsonl");
let w = TelemetryWriter::new(WriterOptions {
telemetry_path: Some(telemetry_path.clone()),
trace_path: None,
capacity: 64,
max_file_bytes: 64, max_generations: 1,
write_delay: None,
write_gate: None,
});
for ts in 0..3u64 {
let mut r = sample_telemetry();
r.ts = ts;
w.enqueue_telemetry(r);
}
let stats = w.shutdown(Duration::from_secs(5)).stats();
assert_eq!(stats.written, 3);
let rotated = tmp.path().join("synaps/api-log.jsonl.1");
assert!(rotated.exists(), "size cap must rotate to .1");
let ts_of = |lines: Vec<String>| -> Vec<u64> {
lines
.iter()
.map(|l| {
serde_json::from_str::<serde_json::Value>(l).unwrap()["ts"]
.as_u64()
.unwrap()
})
.collect()
};
assert_eq!(ts_of(read_lines(&rotated)), vec![1]);
assert_eq!(ts_of(read_lines(&telemetry_path)), vec![2]);
assert!(
!tmp.path().join("synaps/api-log.jsonl.2").exists(),
"the default keeps exactly one generation"
);
}
fn rotating_writer(telemetry_path: &std::path::Path, generations: u32) -> TelemetryWriter {
TelemetryWriter::new(WriterOptions {
telemetry_path: Some(telemetry_path.to_path_buf()),
trace_path: None,
capacity: 1024,
max_file_bytes: 64, max_generations: generations,
write_delay: None,
write_gate: None,
})
}
fn write_n(w: &TelemetryWriter, from: u64, count: u64) {
for ts in from..from + count {
let mut r = sample_telemetry();
r.ts = ts;
w.enqueue_telemetry(r);
}
}
#[test]
fn rotation_prunes_generations_beyond_n_including_after_n_is_lowered() {
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("synaps/api-log.jsonl");
let w = rotating_writer(&path, 5);
write_n(&w, 0, 7);
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
for k in 1..=5 {
assert!(generation_path(&path, k).exists(), ".{k} kept at N=5");
}
std::fs::write(generation_path(&path, 42), b"{\"ts\":999}\n").unwrap();
let w = rotating_writer(&path, 2);
write_n(&w, 100, 1);
let stats = w.shutdown(Duration::from_secs(5)).stats();
assert_eq!((stats.written, stats.io_failures), (1, 0));
assert!(generation_path(&path, 1).exists());
assert!(generation_path(&path, 2).exists());
for k in [3, 4, 5, 6, 42] {
assert!(!generation_path(&path, k).exists(), ".{k} pruned at N=2");
}
}
#[cfg(unix)]
#[test]
fn shifted_generations_are_repaired_to_0600() {
use std::os::unix::fs::PermissionsExt;
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("synaps/api-log.jsonl");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
let old = generation_path(&path, 1);
std::fs::write(&old, b"{\"ts\":1}\n").unwrap();
std::fs::set_permissions(&old, std::fs::Permissions::from_mode(0o644)).unwrap();
let w = rotating_writer(&path, 3);
write_n(&w, 10, 2);
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
for k in 1..=2 {
let mode = std::fs::metadata(generation_path(&path, k)).unwrap().permissions().mode();
assert_eq!(mode & 0o777, 0o600, ".{k}");
}
}
#[test]
fn concurrent_writers_on_one_log_never_double_rotate() {
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("synaps/api-log.jsonl");
const WRITERS: u64 = 4;
const EACH: u64 = 20; let writers: Vec<_> = (0..WRITERS).map(|_| rotating_writer(&path, MAX_GENERATIONS)).collect();
let threads: Vec<_> = writers
.iter()
.enumerate()
.map(|(i, w)| {
let w = w.clone();
std::thread::spawn(move || write_n(&w, i as u64 * 1000, EACH))
})
.collect();
for t in threads {
t.join().unwrap();
}
for w in &writers {
assert!(w.shutdown(Duration::from_secs(10)).is_flushed());
}
let total_written: u64 = writers.iter().map(|w| w.stats().written).sum();
assert_eq!(total_written, WRITERS * EACH);
let mut on_disk = read_lines(&path).len() as u64;
let mut k = 1;
while generation_path(&path, k).exists() {
let n = read_lines(&generation_path(&path, k)).len();
assert!(n > 0, ".{k} is empty: a double rotation");
on_disk += n as u64;
k += 1;
}
let highest = (1..=MAX_GENERATIONS).filter(|k| generation_path(&path, *k).exists()).max().unwrap_or(0);
assert_eq!(highest, k - 1, "generations are contiguous (no gap from a double shift)");
assert_eq!(on_disk, WRITERS * EACH, "no record lost to a double rotation");
}
#[test]
fn rotation_keeps_n_generations_for_both_logs() {
let tmp = tempfile::TempDir::new().unwrap();
let telemetry_path = tmp.path().join("synaps/api-log.jsonl");
let trace_path = tmp.path().join("synaps/request-trace.jsonl");
let w = TelemetryWriter::new(WriterOptions {
telemetry_path: Some(telemetry_path.clone()),
trace_path: Some(trace_path.clone()),
capacity: 64,
max_file_bytes: 64, max_generations: 3,
write_delay: None,
write_gate: None,
});
for ts in 0..6u64 {
let mut r = sample_telemetry();
r.ts = ts;
w.enqueue_telemetry(r);
w.enqueue_trace(sample_trace());
}
let stats = w.shutdown(Duration::from_secs(5)).stats();
assert_eq!(stats.written, 12);
assert_eq!(stats.io_failures, 0);
let ts = |path: PathBuf| -> Vec<u64> {
read_lines(&path)
.iter()
.map(|l| {
serde_json::from_str::<serde_json::Value>(l).unwrap()["ts"]
.as_u64()
.unwrap()
})
.collect()
};
assert_eq!(ts(telemetry_path.clone()), vec![5]);
assert_eq!(ts(generation_path(&telemetry_path, 1)), vec![4]);
assert_eq!(ts(generation_path(&telemetry_path, 2)), vec![3]);
assert_eq!(ts(generation_path(&telemetry_path, 3)), vec![2]);
assert!(!generation_path(&telemetry_path, 4).exists());
for n in 1..=3 {
assert_eq!(read_lines(&generation_path(&trace_path, n)).len(), 1);
}
assert!(!generation_path(&trace_path, 4).exists());
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
for n in 1..=3 {
let mode = std::fs::metadata(generation_path(&telemetry_path, n))
.unwrap()
.permissions()
.mode();
assert_eq!(mode & 0o777, 0o600, "generation {n} stays private");
}
}
}
#[test]
fn shutdown_is_bounded_with_slow_sink() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 32, Some(Duration::from_millis(300)));
for _ in 0..10 {
w.enqueue_telemetry(sample_telemetry());
}
let start = Instant::now();
let out = w.shutdown(Duration::from_millis(200));
let elapsed = start.elapsed();
assert!(!out.is_flushed(), "10×300ms cannot drain in 200ms");
assert!(
elapsed < Duration::from_millis(800),
"shutdown must respect its deadline: {elapsed:?}"
);
}
#[test]
fn shutdown_is_idempotent() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
}
#[tokio::test]
async fn shutdown_async_flushes_off_the_executor() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, telemetry_path, _) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
let out = w.shutdown_async(Duration::from_secs(5)).await;
assert!(out.is_flushed());
assert_eq!(read_lines(&telemetry_path).len(), 1);
}
#[test]
fn enqueue_degrades_on_poisoned_mutex_instead_of_panicking() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, telemetry_path, _) = tmp_writer(tmp.path(), 16, None);
let w2 = w.clone();
let poisoner = std::thread::spawn(move || {
let _guard = w2.shared.tx.lock().unwrap();
panic!("intentional poison for the degradation test");
});
assert!(poisoner.join().is_err(), "poisoner must have panicked");
assert!(w.shared.tx.is_poisoned(), "mutex must be poisoned");
w.enqueue_telemetry(sample_telemetry());
assert_eq!(w.stats().enqueued, 1, "poison recovery keeps intake open");
assert_eq!(w.stats().dropped, 0);
let out = w.shutdown(Duration::from_secs(5));
assert!(out.is_flushed());
assert_eq!(out.stats().written, 1);
assert_eq!(read_lines(&telemetry_path).len(), 1);
}
#[test]
fn fatal_outcome_record_persisted_by_epilogue_flush() {
let tmp = tempfile::TempDir::new().unwrap();
let (w, _, trace_path) = tmp_writer(tmp.path(), 16, None);
let mut trace = sample_trace();
trace.outcome.terminal = agent_core::TurnOutcome::ProviderFailed {
code: "api_error".to_string(),
correlation_id: "corr-fatal-1".to_string(),
};
trace.outcome.http_status = Some(500);
w.enqueue_trace(trace);
let out = w.shutdown(DEFAULT_SHUTDOWN_FLUSH_TIMEOUT);
assert!(out.is_flushed());
let lines = read_lines(&trace_path);
assert_eq!(lines.len(), 1);
let parsed: RequestTrace = serde_json::from_str(&lines[0]).unwrap();
assert!(matches!(
parsed.outcome.terminal,
agent_core::TurnOutcome::ProviderFailed { .. }
));
}
#[cfg(unix)]
mod private_modes {
use super::*;
use serial_test::serial;
use std::os::unix::fs::PermissionsExt;
struct UmaskGuard {
old: libc::mode_t,
}
impl UmaskGuard {
fn set(mask: libc::mode_t) -> Self {
Self {
old: unsafe { libc::umask(mask) },
}
}
}
impl Drop for UmaskGuard {
fn drop(&mut self) {
unsafe {
libc::umask(self.old);
}
}
}
fn mode_of(path: &std::path::Path) -> u32 {
std::fs::metadata(path).unwrap().permissions().mode() & 0o777
}
#[test]
#[serial(umask)]
fn writer_creates_0600_files_and_0700_dir_under_permissive_umask() {
let _umask = UmaskGuard::set(0);
let tmp = tempfile::TempDir::new().unwrap();
let (w, telemetry_path, trace_path) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
w.enqueue_trace(sample_trace());
assert!(w.shutdown(Duration::from_secs(5)).is_flushed());
assert_eq!(mode_of(telemetry_path.parent().unwrap()), 0o700);
assert_eq!(mode_of(&telemetry_path), 0o600);
assert_eq!(mode_of(&trace_path), 0o600);
}
#[test]
fn writer_refuses_symlink_targets() {
let tmp = tempfile::TempDir::new().unwrap();
let dir = tmp.path().join("synaps");
std::fs::create_dir_all(&dir).unwrap();
let victim = tmp.path().join("victim.jsonl");
std::fs::write(&victim, "").unwrap();
std::os::unix::fs::symlink(&victim, dir.join("api-log.jsonl")).unwrap();
std::os::unix::fs::symlink(&victim, dir.join("request-trace.jsonl")).unwrap();
let (w, _, _) = tmp_writer(tmp.path(), 16, None);
w.enqueue_telemetry(sample_telemetry());
w.enqueue_trace(sample_trace());
let stats = w.shutdown(Duration::from_secs(5)).stats();
assert_eq!(
std::fs::read_to_string(&victim).unwrap(),
"",
"no bytes may be written through a planted symlink"
);
assert_eq!(stats.io_failures, 2);
assert_eq!(stats.written, 0);
}
}
}