use std::fs::{File, OpenOptions};
use std::io::{self, IsTerminal, Write};
use std::os::fd::AsRawFd;
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
use std::path::Path;
use std::sync::Once;
use serde::Serialize;
use sha2::{Digest, Sha256};
const MAX_FILE_BYTES: u64 = 1024 * 1024;
#[derive(Serialize)]
struct ErrorEvent {
event_id: String,
timestamp_ms: u64,
version: &'static str,
execution_schema_version: u32,
os: &'static str,
arch: &'static str,
pid: u32,
kind: &'static str,
operation: String,
code: String,
#[serde(skip_serializing_if = "Option::is_none")]
invocation_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
execution_id: Option<String>,
error_sha256: String,
#[serde(skip_serializing_if = "Option::is_none")]
panic_location: Option<String>,
}
impl ErrorEvent {
fn new(
kind: &'static str,
operation: &str,
code: &str,
invocation_id: Option<&str>,
execution_id: Option<&str>,
error: &str,
) -> Self {
Self {
event_id: uuid::Uuid::now_v7().to_string(),
timestamp_ms: crate::model::now_ms(),
version: env!("CARGO_PKG_VERSION"),
execution_schema_version: crate::model::SCHEMA_VERSION,
os: std::env::consts::OS,
arch: std::env::consts::ARCH,
pid: std::process::id(),
kind,
operation: operation.chars().take(96).collect(),
code: code.chars().take(96).collect(),
invocation_id: safe_id(invocation_id),
execution_id: safe_id(execution_id),
error_sha256: format!("{:x}", Sha256::digest(error.as_bytes())),
panic_location: None,
}
}
}
fn safe_id(value: Option<&str>) -> Option<String> {
value
.filter(|value| {
uuid::Uuid::parse_str(value).is_ok()
|| (value.starts_with("exec_")
&& value.len() <= 96
&& value[5..]
.bytes()
.all(|byte| byte.is_ascii_hexdigit() || byte == b'_'))
})
.map(ToString::to_string)
}
fn enabled() -> bool {
!std::env::var("APOC_ERROR_TELEMETRY").is_ok_and(|value| value == "off" || value == "0")
}
pub(crate) fn record(
kind: &'static str,
operation: &str,
code: &str,
invocation_id: Option<&str>,
execution_id: Option<&str>,
error: &str,
) {
if let Ok(home) = crate::store::state_dir() {
record_at(
&home,
kind,
operation,
code,
invocation_id,
execution_id,
error,
);
}
}
pub(crate) fn record_at(
home: &Path,
kind: &'static str,
operation: &str,
code: &str,
invocation_id: Option<&str>,
execution_id: Option<&str>,
error: &str,
) {
if enabled() {
let event = ErrorEvent::new(kind, operation, code, invocation_id, execution_id, error);
let _ = append(home, &event, MAX_FILE_BYTES);
}
}
pub fn record_cli_error(code: &str, error: &str) {
record("cli", "cli", code, None, None, error);
}
pub async fn serve_cli(argv: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
let stdout = io::stdout();
let human = stdout.is_terminal();
let mut writer = FingerprintWriter {
output: stdout,
digest: Sha256::new(),
};
if let Some(code) = crate::build_cli()
.serve_to(argv, &mut writer, human)
.await?
{
if code != 0
&& enabled()
&& let Ok(home) = crate::store::state_dir()
{
let mut event = ErrorEvent::new("cli", "cli", "CLI_EXIT_NONZERO", None, None, "");
event.error_sha256 = format!("{:x}", writer.digest.finalize());
let _ = append(&home, &event, MAX_FILE_BYTES);
}
std::process::exit(code);
}
Ok(())
}
struct FingerprintWriter<W> {
output: W,
digest: Sha256,
}
impl<W: Write> Write for FingerprintWriter<W> {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let written = self.output.write(bytes)?;
self.digest.update(&bytes[..written]);
Ok(written)
}
fn flush(&mut self) -> io::Result<()> {
self.output.flush()
}
}
pub(crate) fn record_rpc_at(
home: &Path,
method: &str,
invocation_id: Option<&str>,
execution_id: Option<&str>,
error: &str,
) {
let code = error
.split_once(':')
.map(|(code, _)| code)
.filter(|code| {
code.bytes()
.all(|byte| byte.is_ascii_uppercase() || byte == b'_')
})
.unwrap_or("DAEMON_TRANSPORT_ERROR");
record_at(
home,
"rpc",
method,
code,
invocation_id,
execution_id,
error,
);
}
fn private_file(path: &Path) -> io::Result<File> {
let file = OpenOptions::new()
.create(true)
.append(true)
.read(true)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK)
.open(path)?;
if !file.metadata()?.is_file() {
return Err(io::Error::other("telemetry path is not a regular file"));
}
file.set_permissions(std::fs::Permissions::from_mode(0o600))?;
Ok(file)
}
fn append(home: &Path, event: &ErrorEvent, max_bytes: u64) -> io::Result<bool> {
let directory = home.join("telemetry");
std::fs::create_dir_all(&directory)?;
if std::fs::symlink_metadata(&directory)?
.file_type()
.is_symlink()
{
return Err(io::Error::other("telemetry directory is a symlink"));
}
std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700))?;
let lock = private_file(&directory.join("errors.lock"))?;
if unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 {
let error = io::Error::last_os_error();
return if error.kind() == io::ErrorKind::WouldBlock {
Ok(false)
} else {
Err(error)
};
}
let mut line = serde_json::to_vec(event)?;
line.push(b'\n');
let path = directory.join("errors.jsonl");
let mut file = private_file(&path)?;
if file.metadata()?.len().saturating_add(line.len() as u64) > max_bytes {
drop(file);
std::fs::rename(&path, directory.join("errors.previous.jsonl"))?;
file = private_file(&path)?;
}
file.write_all(&line)?;
Ok(true)
}
pub fn install_panic_hook() {
static INSTALL: Once = Once::new();
INSTALL.call_once(|| {
let previous = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
if enabled()
&& let Ok(home) = crate::store::state_dir()
{
let message = info
.payload()
.downcast_ref::<&str>()
.copied()
.or_else(|| info.payload().downcast_ref::<String>().map(String::as_str))
.unwrap_or("non-string panic");
let mut event = ErrorEvent::new("panic", "panic", "PANIC", None, None, message);
event.panic_location = info
.location()
.map(|location| format!("{}:{}", location.file(), location.line()));
let _ = append(&home, &event, MAX_FILE_BYTES);
}
previous(info);
}));
});
}
#[cfg(test)]
mod tests {
use super::*;
fn event() -> ErrorEvent {
ErrorEvent::new(
"command",
"execution get",
"EXECUTION_NOT_FOUND",
Some("019f0965-5489-7b86-91a5-ff04a6a2e956"),
Some("exec_0000000000000001_0000000000000002"),
"private-error-canary token=secret-canary",
)
}
#[test]
fn persists_only_metadata_and_a_message_fingerprint() {
let home = tempfile::tempdir().unwrap();
assert!(append(home.path(), &event(), MAX_FILE_BYTES).unwrap());
let bytes = std::fs::read(home.path().join("telemetry/errors.jsonl")).unwrap();
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(value["code"], "EXECUTION_NOT_FOUND");
assert_eq!(
value["error_sha256"],
"726eb749c38ca8e59513581fb4e91e9aa1c324c7fb6176063aebd90188c82fe9"
);
assert_eq!(
value["execution_id"],
"exec_0000000000000001_0000000000000002"
);
assert_eq!(value["error_sha256"].as_str().unwrap().len(), 64);
assert!(!String::from_utf8(bytes).unwrap().contains("canary"));
assert_eq!(
std::fs::metadata(home.path().join("telemetry"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o700
);
assert_eq!(
std::fs::metadata(home.path().join("telemetry/errors.jsonl"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o600
);
}
#[test]
fn rotates_complete_lines_and_retains_only_two_files() {
let home = tempfile::tempdir().unwrap();
let event = event();
let bytes = serde_json::to_vec(&event).unwrap().len() as u64 + 1;
for _ in 0..7 {
assert!(append(home.path(), &event, bytes * 2).unwrap());
}
for name in ["errors.jsonl", "errors.previous.jsonl"] {
let path = home.path().join("telemetry").join(name);
assert!(std::fs::metadata(&path).unwrap().len() <= bytes * 2);
let text = std::fs::read_to_string(path).unwrap();
assert!(text.ends_with('\n'));
for line in text.lines() {
serde_json::from_str::<serde_json::Value>(line).unwrap();
}
}
assert_eq!(
std::fs::read_dir(home.path().join("telemetry"))
.unwrap()
.count(),
3
);
}
#[test]
fn skips_a_contended_lock_instead_of_waiting() {
let home = tempfile::tempdir().unwrap();
append(home.path(), &event(), MAX_FILE_BYTES).unwrap();
let lock = private_file(&home.path().join("telemetry/errors.lock")).unwrap();
assert_eq!(unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) }, 0);
let started = std::time::Instant::now();
assert!(!append(home.path(), &event(), MAX_FILE_BYTES).unwrap());
assert!(started.elapsed() < std::time::Duration::from_millis(100));
}
#[test]
fn does_not_follow_a_log_symlink() {
let home = tempfile::tempdir().unwrap();
let target = home.path().join("protected");
std::fs::write(&target, "unchanged").unwrap();
std::fs::create_dir(home.path().join("telemetry")).unwrap();
std::os::unix::fs::symlink(&target, home.path().join("telemetry/errors.jsonl")).unwrap();
assert!(append(home.path(), &event(), MAX_FILE_BYTES).is_err());
assert_eq!(std::fs::read_to_string(target).unwrap(), "unchanged");
}
#[test]
fn rejects_user_content_as_a_record_identifier() {
assert!(safe_id(Some("secret-canary")).is_none());
assert!(safe_id(Some("exec_canary-with-a-secret")).is_none());
}
#[test]
fn panic_hook_child_probe() {
if std::env::var_os("APOC_TELEMETRY_PANIC_PROBE").is_some() {
install_panic_hook();
panic!("private-panic-canary");
}
}
#[test]
fn panic_hook_records_a_local_fingerprint_and_preserves_the_panic() {
let home = tempfile::tempdir().unwrap();
let runtime = tempfile::tempdir().unwrap();
let data = tempfile::tempdir().unwrap();
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args([
"--exact",
"error_telemetry::tests::panic_hook_child_probe",
"--nocapture",
])
.env("APOC_HOME", home.path())
.env("APOC_RUNTIME_DIR", runtime.path())
.env("XDG_DATA_HOME", data.path())
.env("APOC_TELEMETRY_PANIC_PROBE", "1")
.env_remove("APOC_ERROR_TELEMETRY")
.output()
.unwrap();
assert!(!output.status.success());
assert!(String::from_utf8_lossy(&output.stderr).contains("private-panic-canary"));
let text = std::fs::read_to_string(home.path().join("telemetry/errors.jsonl")).unwrap();
let event: serde_json::Value = serde_json::from_str(&text).unwrap();
assert_eq!(event["kind"], "panic");
assert_eq!(event["code"], "PANIC");
assert!(
event["panic_location"]
.as_str()
.unwrap()
.contains("error_telemetry.rs:")
);
assert!(!text.contains("private-panic-canary"));
}
}