durable-actors 0.18.1

Standalone regional durable-actors control plane, host, and durability runtime
use std::{
    sync::Arc,
    time::{Duration, UNIX_EPOCH},
};

use anyhow::Result;
use opentelemetry::KeyValue;
use opentelemetry::logs::{AnyValue, LogRecord, Logger, LoggerProvider, Severity};
use opentelemetry::trace::{Span, SpanKind, Status, Tracer, TracerProvider};
use opentelemetry_otlp::{LogExporter, SpanExporter, WithExportConfig, WithTonicConfig};
use opentelemetry_sdk::{
    Resource,
    logs::{self, BatchLogProcessor, SdkLogger, SdkLoggerProvider},
    trace::{self, BatchSpanProcessor, SdkTracer, SdkTracerProvider},
};
use tonic::transport::ClientTlsConfig;

use crate::{
    actor::{ActorLogRecord, ActorLogSink, LogLevel, StdioLogSink},
    control_plane::HostAuthorization,
    request_traces::{
        RequestKind,
        otlp::{self, EXPORT_DELAY, LOG_ID, REQUEST_ID},
    },
    request_tracking::{RequestCompletion, RequestObserver, RequestOutcome},
};

const ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
const SCOPE: &str = "durable-actors-host";

pub(crate) struct HostTelemetry {
    traces: SdkTracerProvider,
    logs: SdkLoggerProvider,
}

impl HostTelemetry {
    pub(crate) fn from_env(authorization: HostAuthorization) -> Result<Option<Self>> {
        match std::env::var(ENDPOINT) {
            Ok(endpoint) if !endpoint.is_empty() => {
                Ok(Some(Self::connect(&endpoint, authorization)?))
            }
            _ => Ok(None),
        }
    }

    pub(crate) fn tracer(&self) -> SdkTracer {
        self.traces.tracer(SCOPE)
    }

    pub(crate) fn log_sink(&self) -> Arc<dyn ActorLogSink> {
        Arc::new(OtelLogSink {
            logger: self.logs.logger(SCOPE),
        })
    }

    pub(crate) fn shutdown(&self) {
        if let Err(error) = self.traces.shutdown() {
            tracing::warn!(%error, "request span export did not flush");
        }
        if let Err(error) = self.logs.shutdown() {
            tracing::warn!(%error, "actor log export did not flush");
        }
    }

    fn connect(endpoint: &str, authorization: HostAuthorization) -> Result<Self> {
        let resource = Resource::builder().with_service_name(SCOPE).build();
        let spans = exporter(
            SpanExporter::builder().with_tonic(),
            endpoint,
            &authorization,
        )
        .build()?;
        let logs = exporter(
            LogExporter::builder().with_tonic(),
            endpoint,
            &authorization,
        )
        .build()?;
        Ok(Self {
            traces: SdkTracerProvider::builder()
                .with_resource(resource.clone())
                .with_span_processor(
                    BatchSpanProcessor::builder(spans)
                        .with_batch_config(
                            trace::BatchConfigBuilder::default()
                                .with_scheduled_delay(EXPORT_DELAY)
                                .build(),
                        )
                        .build(),
                )
                .build(),
            logs: SdkLoggerProvider::builder()
                .with_resource(resource)
                .with_log_processor(
                    BatchLogProcessor::builder(logs)
                        .with_batch_config(
                            logs::BatchConfigBuilder::default()
                                .with_scheduled_delay(EXPORT_DELAY)
                                .build(),
                        )
                        .build(),
                )
                .build(),
        })
    }
}

fn exporter<B: WithExportConfig + WithTonicConfig>(
    builder: B,
    endpoint: &str,
    authorization: &HostAuthorization,
) -> B {
    let builder = builder
        .with_endpoint(endpoint)
        .with_interceptor(authorization.clone());
    if endpoint.starts_with("https:") {
        builder.with_tls_config(ClientTlsConfig::new().with_webpki_roots())
    } else {
        builder
    }
}

pub(crate) struct TraceRecorder {
    tracer: SdkTracer,
    operation: String,
    attributes: Vec<KeyValue>,
}

impl TraceRecorder {
    pub(crate) fn new(
        tracer: SdkTracer,
        invocation: &crate::actor::ActorInvocation,
        kind: RequestKind,
        connection_id: Option<String>,
        metadata: Option<serde_json::Value>,
    ) -> Self {
        let mut attributes = vec![
            KeyValue::new(otlp::REQUEST_ID, invocation.request_id.clone()),
            KeyValue::new(otlp::KIND, kind.as_str()),
        ];
        attributes.extend(connection_id.map(|id| KeyValue::new(otlp::CONNECTION_ID, id)));
        attributes.extend(metadata.map(|value| KeyValue::new(otlp::METADATA, value.to_string())));
        Self {
            tracer,
            operation: invocation.method.clone(),
            attributes,
        }
    }
}

impl RequestObserver for TraceRecorder {
    fn record(self: Box<Self>, completion: &RequestCompletion) {
        let started = UNIX_EPOCH + Duration::from_millis(completion.started_at_ms);
        let ended = started + Duration::from_secs_f64(completion.timings.duration_ms / 1000.0);
        let mut attributes = self.attributes;
        attributes.extend(completion_attributes(completion));
        let mut span = self
            .tracer
            .span_builder(self.operation)
            .with_kind(SpanKind::Server)
            .with_start_time(started)
            .with_attributes(attributes)
            .start_with_context(&self.tracer, &opentelemetry::Context::new());
        if matches!(completion.outcome, RequestOutcome::Failed) {
            span.set_status(Status::error(completion.outcome.as_str()));
        }
        span.end_with_timestamp(ended);
    }
}

fn completion_attributes(completion: &RequestCompletion) -> impl Iterator<Item = KeyValue> {
    let timings = &completion.timings;
    [
        Some(KeyValue::new(otlp::OUTCOME, completion.outcome.as_str())),
        Some(KeyValue::new(otlp::HOST_STATE, timings.host_state.as_str())),
        Some(KeyValue::new(otlp::ROUTING_MS, timings.routing_ms)),
        timings
            .queue_wait_ms
            .map(|wait| KeyValue::new(otlp::QUEUE_WAIT_MS, wait)),
        completion
            .state_version
            .and_then(|version| i64::try_from(version).ok())
            .map(|version| KeyValue::new(otlp::STATE_VERSION, version)),
    ]
    .into_iter()
    .flatten()
}

struct OtelLogSink {
    logger: SdkLogger,
}

impl ActorLogSink for OtelLogSink {
    fn emit(&self, record: ActorLogRecord) {
        StdioLogSink::write(&record);
        let mut log = self.logger.create_log_record();
        log.set_timestamp(std::time::SystemTime::now());
        log.set_severity_number(severity(record.level));
        log.set_severity_text(record.level.as_str());
        log.add_attribute(LOG_ID, uuid::Uuid::now_v7().to_string());
        if let Some(request_id) = record.request_id {
            log.add_attribute(REQUEST_ID, request_id);
        }
        log.set_body(AnyValue::from(record.message));
        self.logger.emit(log);
    }
}

fn severity(level: LogLevel) -> Severity {
    match level {
        LogLevel::Debug => Severity::Debug,
        LogLevel::Info => Severity::Info,
        LogLevel::Warn => Severity::Warn,
        LogLevel::Error => Severity::Error,
    }
}

#[cfg(test)]
#[path = "../../tests/unit/host/telemetry.rs"]
mod tests;