Skip to main content

platform_runtime/
functions.rs

1use crate::retries::RetryPolicy;
2use async_trait::async_trait;
3use chrono::{DateTime, Utc};
4use platform_core::{
5    ActorContext, AppError, AppResult, CorrelationId, DbPool, ErrorCode, ExecutionContext,
6    ExecutionId, RuntimeSpanAttributes, TenantId, TraceContext, record_runtime_span_attributes,
7    trace_context_from_headers, trace_headers,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::collections::BTreeMap;
12use std::fmt::Debug;
13use std::sync::Arc;
14use std::time::{Duration, Instant};
15use tracing::Instrument;
16use uuid::Uuid;
17
18const STALE_PROCESSING_LOCK_SECONDS: i64 = 300;
19
20#[async_trait]
21pub trait FunctionHandler: Debug + Send + Sync {
22    async fn call(&self, ctx: ExecutionContext, input: Value) -> AppResult<Value>;
23
24    fn observability(&self) -> Option<FunctionHandlerObservability> {
25        None
26    }
27}
28
29pub use FunctionHandler as RuntimeFunction;
30
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct FunctionHandlerObservability {
33    pub source: String,
34    pub attributes: Value,
35}
36
37impl FunctionHandlerObservability {
38    pub fn new(source: impl Into<String>, attributes: Value) -> Self {
39        Self {
40            source: source.into(),
41            attributes: normalize_log_attributes(attributes),
42        }
43    }
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47enum ExecutionLogSeverity {
48    Info,
49    Error,
50}
51
52impl ExecutionLogSeverity {
53    fn as_str(self) -> &'static str {
54        match self {
55            Self::Info => "info",
56            Self::Error => "error",
57        }
58    }
59}
60
61#[derive(Debug, Clone)]
62pub struct FunctionDefinition {
63    pub name: String,
64    pub version: u16,
65    pub queue: String,
66    pub retry_policy: RetryPolicy,
67    pub handler: Arc<dyn FunctionHandler>,
68}
69
70#[derive(Debug, Default, Clone)]
71pub struct FunctionRegistry {
72    functions: BTreeMap<String, FunctionDefinition>,
73}
74
75impl FunctionRegistry {
76    pub fn register(&mut self, function: FunctionDefinition) {
77        self.functions.insert(function.name.clone(), function);
78    }
79
80    pub fn get(&self, name: &str) -> Option<&FunctionDefinition> {
81        self.functions.get(name)
82    }
83
84    pub fn all(&self) -> impl Iterator<Item = &FunctionDefinition> {
85        self.functions.values()
86    }
87}
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
90#[serde(rename_all = "snake_case")]
91pub enum FunctionRunStatus {
92    Pending,
93    Processing,
94    Completed,
95    Failed,
96    Dead,
97}
98
99impl FunctionRunStatus {
100    fn as_str(self) -> &'static str {
101        match self {
102            Self::Pending => "pending",
103            Self::Processing => "processing",
104            Self::Completed => "completed",
105            Self::Failed => "failed",
106            Self::Dead => "dead",
107        }
108    }
109}
110
111#[derive(Debug, Clone)]
112pub struct EnqueueFunctionRequest {
113    pub function_name: String,
114    pub input_json: Value,
115    pub correlation_id: CorrelationId,
116    pub actor: ActorContext,
117    pub trace: TraceContext,
118    pub causation_id: Option<String>,
119    pub max_attempts: Option<i32>,
120}
121
122#[derive(Debug, Clone, Deserialize, Serialize)]
123pub struct ClaimedFunctionRun {
124    pub id: String,
125    pub function_name: String,
126    pub input_json: Value,
127    pub attempts: i32,
128    pub max_attempts: i32,
129    pub correlation_id: String,
130    pub actor: ActorContext,
131    pub trace: TraceContext,
132    pub causation_id: Option<String>,
133}
134
135#[derive(Debug, Clone)]
136pub struct RuntimeClient {
137    pool: DbPool,
138}
139
140impl RuntimeClient {
141    pub fn new(pool: DbPool) -> Self {
142        Self { pool }
143    }
144
145    pub async fn enqueue_function(&self, request: EnqueueFunctionRequest) -> AppResult<String> {
146        let id = format!("fnrun_{}", Uuid::now_v7());
147        let max_attempts = request.max_attempts.unwrap_or(3);
148        let mut input_json = request.input_json;
149        attach_runtime_context_to_input(
150            &mut input_json,
151            &request.correlation_id,
152            &request.trace,
153            request.causation_id.as_deref(),
154        );
155        let span = tracing::info_span!(
156            "function_enqueue",
157            lenso.correlation_id = tracing::field::Empty,
158            lenso.story_id = tracing::field::Empty,
159            lenso.function_run_id = tracing::field::Empty,
160            lenso.execution.kind = tracing::field::Empty,
161            lenso.execution.name = tracing::field::Empty,
162        );
163        record_runtime_span_attributes(
164            &span,
165            &RuntimeSpanAttributes::function(
166                request.correlation_id.0.clone(),
167                id.clone(),
168                request.function_name.clone(),
169            ),
170        );
171
172        async {
173            sqlx::query(
174                r#"
175            insert into runtime.function_runs (
176                id,
177                function_name,
178                input_json,
179                max_attempts,
180                correlation_id,
181                actor
182            )
183            values ($1, $2, $3, $4, $5, $6)
184            "#,
185            )
186            .bind(&id)
187            .bind(&request.function_name)
188            .bind(&input_json)
189            .bind(max_attempts)
190            .bind(&request.correlation_id.0)
191            .bind(serde_json::to_value(&request.actor).map_err(map_serde_error)?)
192            .execute(&self.pool)
193            .await
194            .map_err(map_runtime_error)
195        }
196        .instrument(span)
197        .await?;
198
199        self.record_function_execution_log(
200            &FunctionLogContext {
201                id: id.clone(),
202                function_name: request.function_name,
203                correlation_id: request.correlation_id.0,
204                trace: request.trace,
205            },
206            ExecutionLogSeverity::Info,
207            "Function run enqueued",
208            serde_json::json!({
209                "attempt": 0,
210                "max_attempts": max_attempts,
211            }),
212        )
213        .await;
214
215        Ok(id)
216    }
217
218    async fn record_function_execution_log(
219        &self,
220        run: &FunctionLogContext,
221        severity: ExecutionLogSeverity,
222        body: &'static str,
223        attributes: Value,
224    ) {
225        emit_function_lifecycle_event(run, severity, body, &attributes, None);
226        if let Err(error) = insert_execution_log_projection(
227            &self.pool,
228            function_log_record(run, severity, body, attributes),
229        )
230        .await
231        {
232            tracing::warn!(
233                error = ?error,
234                function_run_id = %run.id,
235                "failed to write function execution log"
236            );
237        }
238    }
239}
240
241#[derive(Debug, Clone)]
242pub struct RuntimeWorker {
243    pool: DbPool,
244    registry: Arc<FunctionRegistry>,
245    worker_id: String,
246}
247
248impl RuntimeWorker {
249    pub fn new(
250        pool: DbPool,
251        registry: Arc<FunctionRegistry>,
252        worker_id: impl Into<String>,
253    ) -> Self {
254        Self {
255            pool,
256            registry,
257            worker_id: worker_id.into(),
258        }
259    }
260
261    pub async fn claim_batch(&self, batch_size: i64) -> AppResult<Vec<ClaimedFunctionRun>> {
262        let span = tracing::info_span!(
263            "function_claim_batch",
264            worker_id = %self.worker_id,
265            lenso.execution.kind = "function_claim",
266            lenso.execution.name = "function.claim_batch",
267        );
268
269        async {
270            let runs = sqlx::query_as::<_, FunctionRunRow>(
271                r#"
272            with claimed as (
273                select id
274                from runtime.function_runs
275                where (
276                    status in ('pending', 'failed')
277                    and available_at <= now()
278                )
279                or (
280                    status = 'processing'
281                    and locked_at <= now() - ($1::double precision * interval '1 second')
282                )
283                order by available_at asc, created_at asc
284                limit $2
285                for update skip locked
286            )
287            update runtime.function_runs function_run
288            set status = 'processing',
289                locked_at = now(),
290                locked_by = $3,
291                started_at = coalesce(started_at, now()),
292                last_error = null,
293                updated_at = now()
294            from claimed
295            where function_run.id = claimed.id
296            returning
297                function_run.id,
298                function_run.function_name,
299                function_run.input_json,
300                function_run.attempts,
301                function_run.max_attempts,
302                function_run.correlation_id,
303                function_run.actor
304            "#,
305            )
306            .bind(stale_processing_lock_seconds())
307            .bind(batch_size)
308            .bind(&self.worker_id)
309            .fetch_all(&self.pool)
310            .await
311            .map(|rows| {
312                rows.into_iter()
313                    .map(TryInto::try_into)
314                    .collect::<AppResult<Vec<_>>>()
315            })
316            .map_err(map_runtime_error)??;
317
318            for run in &runs {
319                self.record_function_execution_log(
320                    run,
321                    ExecutionLogSeverity::Info,
322                    "Function run claimed",
323                    serde_json::json!({
324                        "attempt": run.attempts + 1,
325                        "max_attempts": run.max_attempts,
326                        "worker_id": self.worker_id,
327                    }),
328                )
329                .await;
330            }
331
332            Ok(runs)
333        }
334        .instrument(span)
335        .await
336    }
337
338    pub async fn claim_and_run_batch(&self, batch_size: i64) -> AppResult<usize> {
339        let span = tracing::info_span!(
340            "function_worker_loop",
341            worker_id = %self.worker_id,
342            lenso.execution.kind = "worker_loop",
343            lenso.execution.name = "runtime_worker.claim_and_run_batch",
344        );
345
346        async {
347            let runs = self.claim_batch(batch_size).await?;
348            let count = runs.len();
349
350            for run in runs {
351                self.run_claimed(run).await?;
352            }
353
354            Ok(count)
355        }
356        .instrument(span)
357        .await
358    }
359
360    async fn run_claimed(&self, run: ClaimedFunctionRun) -> AppResult<()> {
361        let span = tracing::info_span!(
362            "function_run",
363            lenso.correlation_id = tracing::field::Empty,
364            lenso.story_id = tracing::field::Empty,
365            lenso.function_run_id = tracing::field::Empty,
366            lenso.execution.kind = tracing::field::Empty,
367            lenso.execution.name = tracing::field::Empty,
368        );
369        record_runtime_span_attributes(
370            &span,
371            &RuntimeSpanAttributes::function(
372                run.correlation_id.clone(),
373                run.id.clone(),
374                run.function_name.clone(),
375            ),
376        );
377
378        async {
379            self.record_function_execution_log(
380                &run,
381                ExecutionLogSeverity::Info,
382                "Function run started",
383                serde_json::json!({
384                    "attempt": run.attempts + 1,
385                    "max_attempts": run.max_attempts,
386                    "worker_id": self.worker_id,
387                }),
388            )
389            .await;
390
391            let Some(definition) = self.registry.get(&run.function_name) else {
392                let error = AppError::new(
393                    ErrorCode::Internal,
394                    format!("Runtime function {} is not registered", run.function_name),
395                )
396                .retryable();
397                self.mark_failed(&run, &error, RetryPolicy::default().initial_delay)
398                    .await?;
399                return Ok(());
400            };
401
402            let attempt = u32::try_from(run.attempts + 1).unwrap_or(u32::MAX);
403            let ctx = ExecutionContext {
404                execution_id: ExecutionId(run.id.clone()),
405                function_name: run.function_name.clone(),
406                attempt,
407                queue: definition.queue.clone(),
408                correlation_id: CorrelationId::new(run.correlation_id.clone()),
409                causation_id: run.causation_id.clone(),
410                actor: run.actor.clone(),
411                tenant_id: None::<TenantId>,
412                trace: run.trace.clone(),
413                deadline: None::<DateTime<Utc>>,
414            };
415
416            let observability = definition.handler.observability();
417            let started_at = Utc::now();
418            let started = Instant::now();
419            let result = definition.handler.call(ctx, run.input_json.clone()).await;
420            let duration_ms = started.elapsed().as_millis().try_into().unwrap_or(i64::MAX);
421            if let Some(observability) = observability {
422                self.record_function_handler_operation_log(
423                    &run,
424                    observability,
425                    started_at,
426                    duration_ms,
427                    result.as_ref().err(),
428                )
429                .await;
430            }
431
432            match result {
433                Ok(_output) => self.mark_completed(&run).await,
434                Err(error) => {
435                    self.mark_failed(&run, &error, definition.retry_policy.initial_delay)
436                        .await
437                }
438            }
439        }
440        .instrument(span)
441        .await
442    }
443
444    pub async fn mark_completed(&self, run: &ClaimedFunctionRun) -> AppResult<()> {
445        sqlx::query(
446            r#"
447            update runtime.function_runs
448            set status = 'completed',
449                completed_at = now(),
450                locked_at = null,
451                locked_by = null,
452                last_error = null,
453                updated_at = now()
454            where id = $1
455            "#,
456        )
457        .bind(&run.id)
458        .execute(&self.pool)
459        .await
460        .map(|_| ())
461        .map_err(map_runtime_error)?;
462
463        self.record_function_execution_log(
464            run,
465            ExecutionLogSeverity::Info,
466            "Function run completed",
467            serde_json::json!({
468                "attempt": run.attempts + 1,
469                "max_attempts": run.max_attempts,
470                "worker_id": self.worker_id,
471            }),
472        )
473        .await;
474
475        Ok(())
476    }
477
478    pub async fn mark_failed(
479        &self,
480        run: &ClaimedFunctionRun,
481        error: &AppError,
482        retry_delay: Duration,
483    ) -> AppResult<()> {
484        let next_attempt = run.attempts + 1;
485        let status = if next_attempt >= run.max_attempts {
486            FunctionRunStatus::Dead
487        } else if error.retryable {
488            FunctionRunStatus::Failed
489        } else {
490            FunctionRunStatus::Dead
491        };
492
493        let span = tracing::info_span!(
494            "function_run_fail",
495            lenso.correlation_id = tracing::field::Empty,
496            lenso.story_id = tracing::field::Empty,
497            lenso.function_run_id = tracing::field::Empty,
498            lenso.execution.kind = tracing::field::Empty,
499            lenso.execution.name = tracing::field::Empty,
500        );
501        record_runtime_span_attributes(
502            &span,
503            &RuntimeSpanAttributes::function(
504                run.correlation_id.clone(),
505                run.id.clone(),
506                run.function_name.clone(),
507            ),
508        );
509
510        async {
511            sqlx::query(
512                r#"
513            update runtime.function_runs
514            set status = $2,
515                attempts = attempts + 1,
516                available_at = case
517                    when $2 = 'failed' then now() + ($4::double precision * interval '1 second')
518                    else available_at
519                end,
520                locked_at = null,
521                locked_by = null,
522                last_error = $3,
523                updated_at = now()
524            where id = $1
525            "#,
526            )
527            .bind(&run.id)
528            .bind(status.as_str())
529            .bind(error.public_message.as_str())
530            .bind(retry_delay_seconds(retry_delay))
531            .execute(&self.pool)
532            .await
533            .map(|_| ())
534            .map_err(map_runtime_error)?;
535
536            self.record_function_execution_log(
537                run,
538                ExecutionLogSeverity::Error,
539                if status == FunctionRunStatus::Dead {
540                    "Function run marked dead"
541                } else {
542                    "Function run failed"
543                },
544                serde_json::json!({
545                    "attempt": next_attempt,
546                    "max_attempts": run.max_attempts,
547                    "status": status.as_str(),
548                    "retryable": error.retryable,
549                    "error": error.public_message,
550                    "worker_id": self.worker_id,
551                }),
552            )
553            .await;
554
555            Ok(())
556        }
557        .instrument(span)
558        .await
559    }
560
561    async fn record_function_execution_log(
562        &self,
563        run: &ClaimedFunctionRun,
564        severity: ExecutionLogSeverity,
565        body: &'static str,
566        attributes: Value,
567    ) {
568        emit_function_lifecycle_event(run, severity, body, &attributes, Some(&self.worker_id));
569        if let Err(error) = insert_execution_log_projection(
570            &self.pool,
571            function_log_record(run, severity, body, attributes),
572        )
573        .await
574        {
575            tracing::warn!(
576                error = ?error,
577                function_run_id = %run.id,
578                "failed to write function execution log"
579            );
580        }
581    }
582
583    async fn record_function_handler_operation_log(
584        &self,
585        run: &ClaimedFunctionRun,
586        observability: FunctionHandlerObservability,
587        started_at: DateTime<Utc>,
588        duration_ms: i64,
589        error: Option<&AppError>,
590    ) {
591        let body = if error.is_some() {
592            "Function handler operation failed"
593        } else {
594            "Function handler operation completed"
595        };
596        let severity = if error.is_some() {
597            ExecutionLogSeverity::Error
598        } else {
599            ExecutionLogSeverity::Info
600        };
601        let attributes = function_handler_operation_attributes(
602            run,
603            &self.worker_id,
604            observability,
605            duration_ms,
606            error,
607        );
608        emit_function_lifecycle_event(run, severity, body, &attributes, Some(&self.worker_id));
609        if let Err(error) = insert_execution_log_projection(
610            &self.pool,
611            function_log_record(run, severity, body, attributes).with_occurred_at(started_at),
612        )
613        .await
614        {
615            tracing::warn!(
616                error = ?error,
617                function_run_id = %run.id,
618                "failed to write function handler operation log"
619            );
620        }
621    }
622}
623
624#[derive(Debug)]
625struct FunctionLogContext {
626    id: String,
627    function_name: String,
628    correlation_id: String,
629    trace: TraceContext,
630}
631
632trait FunctionLogSource {
633    fn id(&self) -> String;
634    fn function_name(&self) -> String;
635    fn correlation_id(&self) -> String;
636    fn trace(&self) -> TraceContext;
637}
638
639impl FunctionLogSource for FunctionLogContext {
640    fn id(&self) -> String {
641        self.id.clone()
642    }
643
644    fn function_name(&self) -> String {
645        self.function_name.clone()
646    }
647
648    fn correlation_id(&self) -> String {
649        self.correlation_id.clone()
650    }
651
652    fn trace(&self) -> TraceContext {
653        self.trace.clone()
654    }
655}
656
657impl FunctionLogSource for ClaimedFunctionRun {
658    fn id(&self) -> String {
659        self.id.clone()
660    }
661
662    fn function_name(&self) -> String {
663        self.function_name.clone()
664    }
665
666    fn correlation_id(&self) -> String {
667        self.correlation_id.clone()
668    }
669
670    fn trace(&self) -> TraceContext {
671        self.trace.clone()
672    }
673}
674
675fn emit_function_lifecycle_event(
676    run: &impl FunctionLogSource,
677    severity: ExecutionLogSeverity,
678    body: &'static str,
679    attributes: &Value,
680    worker_id: Option<&str>,
681) {
682    match severity {
683        ExecutionLogSeverity::Error => {
684            tracing::error!(
685                function_run_id = %run.id(),
686                function_name = %run.function_name(),
687                correlation_id = %run.correlation_id(),
688                worker_id = worker_id.unwrap_or(""),
689                attributes = %attributes,
690                "{body}"
691            );
692        }
693        _ => {
694            tracing::info!(
695                function_run_id = %run.id(),
696                function_name = %run.function_name(),
697                correlation_id = %run.correlation_id(),
698                worker_id = worker_id.unwrap_or(""),
699                attributes = %attributes,
700                "{body}"
701            );
702        }
703    }
704}
705
706fn function_log_record(
707    run: &impl FunctionLogSource,
708    severity: ExecutionLogSeverity,
709    body: impl Into<String>,
710    attributes: Value,
711) -> ExecutionLogProjectionRecord {
712    ExecutionLogProjectionRecord::from_runtime_attrs(
713        RuntimeSpanAttributes::function(run.correlation_id(), run.id(), run.function_name()),
714        severity,
715        body,
716    )
717    .with_attributes(attributes)
718    .with_trace(run.trace())
719}
720
721#[derive(Debug, Clone)]
722struct ExecutionLogProjectionRecord {
723    correlation_id: String,
724    execution_id: String,
725    execution_type: String,
726    execution_name: String,
727    severity: ExecutionLogSeverity,
728    body: String,
729    attributes: Value,
730    trace: TraceContext,
731    service_name: String,
732    occurred_at: Option<DateTime<Utc>>,
733}
734
735impl ExecutionLogProjectionRecord {
736    fn from_runtime_attrs(
737        attrs: RuntimeSpanAttributes,
738        severity: ExecutionLogSeverity,
739        body: impl Into<String>,
740    ) -> Self {
741        let execution_id = attrs
742            .function_run_id
743            .clone()
744            .or(attrs.outbox_event_id)
745            .unwrap_or_else(|| attrs.story_id.clone());
746
747        Self {
748            correlation_id: attrs.correlation_id,
749            execution_id,
750            execution_type: attrs.execution_kind,
751            execution_name: attrs.execution_name,
752            severity,
753            body: body.into(),
754            attributes: Value::Object(Default::default()),
755            trace: TraceContext::default(),
756            service_name: "lenso".to_owned(),
757            occurred_at: None,
758        }
759    }
760
761    fn with_attributes(mut self, attributes: Value) -> Self {
762        self.attributes = attributes;
763        self
764    }
765
766    fn with_trace(mut self, trace: TraceContext) -> Self {
767        self.trace = trace;
768        self
769    }
770
771    fn with_occurred_at(mut self, occurred_at: DateTime<Utc>) -> Self {
772        self.occurred_at = Some(occurred_at);
773        self
774    }
775}
776
777async fn insert_execution_log_projection(
778    pool: &DbPool,
779    record: ExecutionLogProjectionRecord,
780) -> AppResult<String> {
781    let id = format!("elog_{}", Uuid::now_v7());
782    let occurred_at = record.occurred_at.unwrap_or_else(Utc::now);
783
784    sqlx::query(
785        r#"
786        insert into platform.execution_logs (
787            id,
788            correlation_id,
789            story_id,
790            execution_id,
791            execution_type,
792            execution_name,
793            occurred_at,
794            severity,
795            body,
796            attributes,
797            trace_id,
798            span_id,
799            service_name,
800            redacted_fields
801        )
802        values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
803        "#,
804    )
805    .bind(&id)
806    .bind(&record.correlation_id)
807    .bind(&record.correlation_id)
808    .bind(&record.execution_id)
809    .bind(&record.execution_type)
810    .bind(&record.execution_name)
811    .bind(occurred_at)
812    .bind(record.severity.as_str())
813    .bind(&record.body)
814    .bind(normalize_log_attributes(record.attributes))
815    .bind(&record.trace.trace_id)
816    .bind(&record.trace.span_id)
817    .bind(&record.service_name)
818    .bind(Vec::<String>::new())
819    .execute(pool)
820    .await
821    .map_err(map_runtime_error)?;
822
823    Ok(id)
824}
825
826fn function_handler_operation_attributes(
827    run: &ClaimedFunctionRun,
828    worker_id: &str,
829    observability: FunctionHandlerObservability,
830    duration_ms: i64,
831    error: Option<&AppError>,
832) -> Value {
833    let mut attributes = match observability.attributes {
834        Value::Object(attributes) => attributes,
835        other => serde_json::Map::from_iter([("value".to_owned(), other)]),
836    };
837    attributes.insert("source".to_owned(), Value::String(observability.source));
838    attributes.insert("attempt".to_owned(), serde_json::json!(run.attempts + 1));
839    attributes.insert(
840        "max_attempts".to_owned(),
841        serde_json::json!(run.max_attempts),
842    );
843    attributes.insert("duration_ms".to_owned(), serde_json::json!(duration_ms));
844    attributes.insert("success".to_owned(), serde_json::json!(error.is_none()));
845    attributes.insert("worker_id".to_owned(), serde_json::json!(worker_id));
846    attributes.insert(
847        "function_name".to_owned(),
848        serde_json::json!(run.function_name),
849    );
850    attributes.insert("request_id".to_owned(), serde_json::json!(run.id));
851    attributes.insert("trace_id".to_owned(), serde_json::json!(run.trace.trace_id));
852    attributes.insert("span_id".to_owned(), serde_json::json!(run.trace.span_id));
853
854    if let Some(error) = error {
855        attributes.insert(
856            "error_code".to_owned(),
857            serde_json::json!(error.code.as_str()),
858        );
859        attributes.insert("error".to_owned(), serde_json::json!(error.public_message));
860        attributes.insert("retryable".to_owned(), serde_json::json!(error.retryable));
861        attributes.insert("error_details".to_owned(), serde_json::json!(error.details));
862    }
863
864    Value::Object(attributes)
865}
866
867fn normalize_log_attributes(attributes: Value) -> Value {
868    match attributes {
869        Value::Object(_) => attributes,
870        other => serde_json::json!({ "value": other }),
871    }
872}
873
874fn stale_processing_lock_seconds() -> f64 {
875    STALE_PROCESSING_LOCK_SECONDS as f64
876}
877
878fn retry_delay_seconds(delay: Duration) -> f64 {
879    delay.as_secs_f64()
880}
881
882type FunctionRunRow = (String, String, Value, i32, i32, String, Value);
883
884impl TryFrom<FunctionRunRow> for ClaimedFunctionRun {
885    type Error = AppError;
886
887    fn try_from(row: FunctionRunRow) -> Result<Self, Self::Error> {
888        let (id, function_name, input_json, attempts, max_attempts, correlation_id, actor) = row;
889        let runtime_context = input_json.get("_lenso_runtime");
890        let trace = runtime_context
891            .map(trace_context_from_headers)
892            .unwrap_or_default();
893        let causation_id = runtime_context
894            .and_then(|context| context.get("causation_id"))
895            .and_then(Value::as_str)
896            .map(ToOwned::to_owned);
897        Ok(Self {
898            id,
899            function_name,
900            input_json,
901            attempts,
902            max_attempts,
903            correlation_id,
904            actor: serde_json::from_value(actor).map_err(map_serde_error)?,
905            trace,
906            causation_id,
907        })
908    }
909}
910
911fn attach_runtime_context_to_input(
912    input_json: &mut Value,
913    correlation_id: &CorrelationId,
914    trace: &TraceContext,
915    causation_id: Option<&str>,
916) {
917    let mut runtime_context = trace_headers(trace, correlation_id);
918    if let Some(causation_id) = causation_id {
919        runtime_context["causation_id"] = Value::String(causation_id.to_owned());
920    }
921
922    match input_json {
923        Value::Object(object) => {
924            object.insert("_lenso_runtime".to_owned(), runtime_context);
925        }
926        other => {
927            *other = serde_json::json!({
928                "payload": other.clone(),
929                "_lenso_runtime": runtime_context,
930            });
931        }
932    }
933}
934
935fn map_runtime_error(source: sqlx::Error) -> AppError {
936    AppError::new(ErrorCode::Internal, "Runtime operation failed").with_source(source)
937}
938
939fn map_serde_error(source: serde_json::Error) -> AppError {
940    AppError::new(ErrorCode::Internal, "Runtime payload serialization failed").with_source(source)
941}