durable-actors 0.16.0

Standalone regional durable-actors control plane, host, and durability runtime
use crate::control_plane::admin::validate_component;
use crate::request_tracking::HostState;
use anyhow::{Context, Result};
use opentelemetry::{
    KeyValue,
    trace::{Span, SpanKind, Status, Tracer},
};
use opentelemetry_sdk::trace::SdkTracer;
use serde::{Deserialize, Serialize};
use std::{
    sync::Arc,
    time::{Duration, UNIX_EPOCH},
};

pub(crate) use crate::request_tracking::RequestOutcome;
use crate::request_tracking::{RequestCompletion, RequestObserver};

pub(crate) mod history;
pub(crate) mod live;
pub(crate) mod logs;
pub(crate) mod metrics;
pub(crate) mod otlp;
pub(crate) mod persistence;
pub(crate) mod replay;
use live::{InProcessFeed, LiveFeed, LiveStream};
use persistence::{TraceQueries, sqlite::SqliteTracePersistence};
use replay::ReplayQuery;

pub(crate) const TRACE_CAPACITY: usize = 500;

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RequestTrace {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state_version: Option<u64>,
    pub project_id: String,
    pub request_id: String,
    pub actor_name: String,
    pub actor_id: String,
    pub kind: RequestKind,
    pub operation: String,
    pub connection_id: Option<String>,
    pub started_at_ms: u64,
    pub routing_ms: f64,
    pub duration_ms: f64,
    pub queue_wait_ms: Option<f64>,
    #[serde(default)]
    pub host_state: HostState,
    pub outcome: RequestOutcome,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub metadata: Option<serde_json::Value>,
}

#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum RequestKind {
    Method,
    Websocket,
}

impl RequestKind {
    pub(crate) fn as_str(self) -> &'static str {
        match self {
            Self::Method => "method",
            Self::Websocket => "websocket",
        }
    }
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TraceEvent {
    #[serde(default = "new_event_id")]
    pub event_id: String,
    pub host_id: String,
    pub session_id: String,
    #[serde(flatten)]
    pub trace: RequestTrace,
}

fn new_event_id() -> String {
    uuid::Uuid::new_v4().to_string()
}

#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TraceRecord {
    pub sequence: u64,
    #[serde(flatten)]
    pub event: TraceEvent,
}

#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TracePage {
    pub epoch: String,
    pub cursor: u64,
    pub capacity: usize,
    pub records: Vec<TraceRecord>,
    pub next_cursor: Option<String>,
    pub resume_cursor: String,
    pub reset: bool,
    #[serde(skip_serializing_if = "std::ops::Not::not")]
    pub snapshot: bool,
}

impl TracePage {
    pub(crate) fn empty(capacity: usize) -> Self {
        Self {
            epoch: "none".into(),
            cursor: 0,
            capacity,
            records: Vec::new(),
            next_cursor: None,
            resume_cursor: "snapshot".into(),
            reset: false,
            snapshot: false,
        }
    }
}

#[derive(Debug)]
pub(crate) struct InvalidTraceQuery;

impl std::fmt::Display for InvalidTraceQuery {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter.write_str("Invalid or incompatible request trace query")
    }
}
impl std::error::Error for InvalidTraceQuery {}

#[derive(Clone)]
pub(crate) struct TraceStore {
    queries: Arc<dyn TraceQueries>,
    live: Arc<dyn LiveFeed>,
}

impl Default for TraceStore {
    fn default() -> Self {
        Self::new(
            Arc::new(SqliteTracePersistence::in_memory()),
            Arc::new(InProcessFeed::default()),
        )
    }
}

impl TraceStore {
    pub(crate) fn new(queries: Arc<dyn TraceQueries>, live: Arc<dyn LiveFeed>) -> Self {
        Self { queries, live }
    }

    pub(crate) async fn subscribe(
        &self,
        project_id: &str,
        signal: otlp::Signal,
    ) -> Result<LiveStream> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        self.live
            .subscribe(otlp::live_channel(project_id, signal))
            .await
    }

    pub(crate) async fn replay(&self, project_id: &str, query: &ReplayQuery) -> Result<TracePage> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.replay(project_id, query).await
    }

    pub(crate) async fn history(
        &self,
        project_id: &str,
        query: &history::HistoryQuery,
    ) -> Result<TracePage> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.history(project_id, query).await
    }

    pub(crate) async fn logs(
        &self,
        project_id: &str,
        query: &logs::LogQuery,
    ) -> Result<logs::LogPage> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.logs(project_id, query).await
    }

    pub(crate) async fn log_histogram(
        &self,
        project_id: &str,
        query: &logs::LogHistogramQuery,
    ) -> Result<logs::LogHistogram> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.log_histogram(project_id, query).await
    }

    pub(crate) async fn metrics(
        &self,
        project_id: &str,
        query: &metrics::TimeRange,
    ) -> Result<metrics::OverviewMetrics> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.metrics(project_id, query).await
    }

    pub(crate) async fn queue_waits(
        &self,
        project_id: &str,
        query: &metrics::QueueWaitQuery,
    ) -> Result<Vec<metrics::QueueWaitRow>> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.queue_waits(project_id, query).await
    }

    pub(crate) async fn websockets(
        &self,
        project_id: &str,
        query: &metrics::TimeRange,
    ) -> Result<Vec<metrics::SocketSession>> {
        validate_component("project ID", project_id, 64).context(InvalidTraceQuery)?;
        query.validate().context(InvalidTraceQuery)?;
        self.queries.websockets(project_id, query).await
    }
}

pub(crate) struct TraceRecorder {
    tracer: SdkTracer,
    operation: String,
    attributes: Vec<KeyValue>,
}

impl TraceRecorder {
    pub(crate) fn new(
        tracer: SdkTracer,
        invocation: &crate::actor::ActorInvocation,
        kind: RequestKind,
        connection_id: Option<String>,
        metadata: Option<serde_json::Value>,
    ) -> Self {
        let mut attributes = vec![
            KeyValue::new(otlp::REQUEST_ID, invocation.request_id.clone()),
            KeyValue::new(otlp::KIND, kind.as_str()),
        ];
        attributes.extend(connection_id.map(|id| KeyValue::new(otlp::CONNECTION_ID, id)));
        attributes.extend(metadata.map(|value| KeyValue::new(otlp::METADATA, value.to_string())));
        Self {
            tracer,
            operation: invocation.method.clone(),
            attributes,
        }
    }
}

impl RequestObserver for TraceRecorder {
    fn record(self: Box<Self>, completion: &RequestCompletion) {
        let started = UNIX_EPOCH + Duration::from_millis(completion.started_at_ms);
        let ended = started + Duration::from_secs_f64(completion.timings.duration_ms / 1000.0);
        let mut attributes = self.attributes;
        attributes.extend(completion_attributes(completion));
        let mut span = self
            .tracer
            .span_builder(self.operation)
            .with_kind(SpanKind::Server)
            .with_start_time(started)
            .with_attributes(attributes)
            .start_with_context(&self.tracer, &opentelemetry::Context::new());
        if matches!(completion.outcome, RequestOutcome::Failed) {
            span.set_status(Status::error(completion.outcome.as_str()));
        }
        span.end_with_timestamp(ended);
    }
}

fn completion_attributes(completion: &RequestCompletion) -> impl Iterator<Item = KeyValue> {
    let timings = &completion.timings;
    [
        Some(KeyValue::new(otlp::OUTCOME, completion.outcome.as_str())),
        Some(KeyValue::new(otlp::HOST_STATE, timings.host_state.as_str())),
        Some(KeyValue::new(otlp::ROUTING_MS, timings.routing_ms)),
        timings
            .queue_wait_ms
            .map(|wait| KeyValue::new(otlp::QUEUE_WAIT_MS, wait)),
        completion
            .state_version
            .and_then(|version| i64::try_from(version).ok())
            .map(|version| KeyValue::new(otlp::STATE_VERSION, version)),
    ]
    .into_iter()
    .flatten()
}

#[cfg(test)]
#[path = "../tests/unit/request_traces/tests.rs"]
mod tests;