1use crate::retries::RetryPolicy;
2use async_trait::async_trait;
3use chrono::{DateTime, Utc};
4use platform_core::{
5 ActorContext, AppError, AppResult, CorrelationId, DbPool, ErrorCode, ExecutionContext,
6 ExecutionId, RuntimeSpanAttributes, TenantId, TraceContext, record_runtime_span_attributes,
7 trace_context_from_headers, trace_headers,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::collections::BTreeMap;
12use std::fmt::Debug;
13use std::sync::Arc;
14use std::time::{Duration, Instant};
15use tracing::Instrument;
16use uuid::Uuid;
17
18const STALE_PROCESSING_LOCK_SECONDS: i64 = 300;
19
20#[async_trait]
21pub trait FunctionHandler: Debug + Send + Sync {
22 async fn call(&self, ctx: ExecutionContext, input: Value) -> AppResult<Value>;
23
24 fn observability(&self) -> Option<FunctionHandlerObservability> {
25 None
26 }
27}
28
29pub use FunctionHandler as RuntimeFunction;
30
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct FunctionHandlerObservability {
33 pub source: String,
34 pub attributes: Value,
35}
36
37impl FunctionHandlerObservability {
38 pub fn new(source: impl Into<String>, attributes: Value) -> Self {
39 Self {
40 source: source.into(),
41 attributes: normalize_log_attributes(attributes),
42 }
43 }
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47enum ExecutionLogSeverity {
48 Info,
49 Error,
50}
51
52impl ExecutionLogSeverity {
53 fn as_str(self) -> &'static str {
54 match self {
55 Self::Info => "info",
56 Self::Error => "error",
57 }
58 }
59}
60
61#[derive(Debug, Clone)]
62pub struct FunctionDefinition {
63 pub name: String,
64 pub version: u16,
65 pub queue: String,
66 pub retry_policy: RetryPolicy,
67 pub handler: Arc<dyn FunctionHandler>,
68}
69
70#[derive(Debug, Default, Clone)]
71pub struct FunctionRegistry {
72 functions: BTreeMap<String, FunctionDefinition>,
73}
74
75impl FunctionRegistry {
76 pub fn register(&mut self, function: FunctionDefinition) {
77 self.functions.insert(function.name.clone(), function);
78 }
79
80 pub fn get(&self, name: &str) -> Option<&FunctionDefinition> {
81 self.functions.get(name)
82 }
83
84 pub fn all(&self) -> impl Iterator<Item = &FunctionDefinition> {
85 self.functions.values()
86 }
87}
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
90#[serde(rename_all = "snake_case")]
91pub enum FunctionRunStatus {
92 Pending,
93 Processing,
94 Completed,
95 Failed,
96 Dead,
97}
98
99impl FunctionRunStatus {
100 fn as_str(self) -> &'static str {
101 match self {
102 Self::Pending => "pending",
103 Self::Processing => "processing",
104 Self::Completed => "completed",
105 Self::Failed => "failed",
106 Self::Dead => "dead",
107 }
108 }
109}
110
111#[derive(Debug, Clone)]
112pub struct EnqueueFunctionRequest {
113 pub function_name: String,
114 pub input_json: Value,
115 pub correlation_id: CorrelationId,
116 pub actor: ActorContext,
117 pub trace: TraceContext,
118 pub causation_id: Option<String>,
119 pub max_attempts: Option<i32>,
120}
121
122#[derive(Debug, Clone, Deserialize, Serialize)]
123pub struct ClaimedFunctionRun {
124 pub id: String,
125 pub function_name: String,
126 pub input_json: Value,
127 pub attempts: i32,
128 pub max_attempts: i32,
129 pub correlation_id: String,
130 pub actor: ActorContext,
131 pub trace: TraceContext,
132 pub causation_id: Option<String>,
133}
134
135#[derive(Debug, Clone)]
136pub struct RuntimeClient {
137 pool: DbPool,
138}
139
140impl RuntimeClient {
141 pub fn new(pool: DbPool) -> Self {
142 Self { pool }
143 }
144
145 pub async fn enqueue_function(&self, request: EnqueueFunctionRequest) -> AppResult<String> {
146 let id = format!("fnrun_{}", Uuid::now_v7());
147 let max_attempts = request.max_attempts.unwrap_or(3);
148 let mut input_json = request.input_json;
149 attach_runtime_context_to_input(
150 &mut input_json,
151 &request.correlation_id,
152 &request.trace,
153 request.causation_id.as_deref(),
154 );
155 let span = tracing::info_span!(
156 "function_enqueue",
157 lenso.correlation_id = tracing::field::Empty,
158 lenso.story_id = tracing::field::Empty,
159 lenso.function_run_id = tracing::field::Empty,
160 lenso.execution.kind = tracing::field::Empty,
161 lenso.execution.name = tracing::field::Empty,
162 );
163 record_runtime_span_attributes(
164 &span,
165 &RuntimeSpanAttributes::function(
166 request.correlation_id.0.clone(),
167 id.clone(),
168 request.function_name.clone(),
169 ),
170 );
171
172 async {
173 sqlx::query(
174 r#"
175 insert into runtime.function_runs (
176 id,
177 function_name,
178 input_json,
179 max_attempts,
180 correlation_id,
181 actor
182 )
183 values ($1, $2, $3, $4, $5, $6)
184 "#,
185 )
186 .bind(&id)
187 .bind(&request.function_name)
188 .bind(&input_json)
189 .bind(max_attempts)
190 .bind(&request.correlation_id.0)
191 .bind(serde_json::to_value(&request.actor).map_err(map_serde_error)?)
192 .execute(&self.pool)
193 .await
194 .map_err(map_runtime_error)
195 }
196 .instrument(span)
197 .await?;
198
199 self.record_function_execution_log(
200 &FunctionLogContext {
201 id: id.clone(),
202 function_name: request.function_name,
203 correlation_id: request.correlation_id.0,
204 trace: request.trace,
205 },
206 ExecutionLogSeverity::Info,
207 "Function run enqueued",
208 serde_json::json!({
209 "attempt": 0,
210 "max_attempts": max_attempts,
211 }),
212 )
213 .await;
214
215 Ok(id)
216 }
217
218 async fn record_function_execution_log(
219 &self,
220 run: &FunctionLogContext,
221 severity: ExecutionLogSeverity,
222 body: &'static str,
223 attributes: Value,
224 ) {
225 emit_function_lifecycle_event(run, severity, body, &attributes, None);
226 if let Err(error) = insert_execution_log_projection(
227 &self.pool,
228 function_log_record(run, severity, body, attributes),
229 )
230 .await
231 {
232 tracing::warn!(
233 error = ?error,
234 function_run_id = %run.id,
235 "failed to write function execution log"
236 );
237 }
238 }
239}
240
241#[derive(Debug, Clone)]
242pub struct RuntimeWorker {
243 pool: DbPool,
244 registry: Arc<FunctionRegistry>,
245 worker_id: String,
246}
247
248impl RuntimeWorker {
249 pub fn new(
250 pool: DbPool,
251 registry: Arc<FunctionRegistry>,
252 worker_id: impl Into<String>,
253 ) -> Self {
254 Self {
255 pool,
256 registry,
257 worker_id: worker_id.into(),
258 }
259 }
260
261 pub async fn claim_batch(&self, batch_size: i64) -> AppResult<Vec<ClaimedFunctionRun>> {
262 let span = tracing::info_span!(
263 "function_claim_batch",
264 worker_id = %self.worker_id,
265 lenso.execution.kind = "function_claim",
266 lenso.execution.name = "function.claim_batch",
267 );
268
269 async {
270 let runs = sqlx::query_as::<_, FunctionRunRow>(
271 r#"
272 with claimed as (
273 select id
274 from runtime.function_runs
275 where (
276 status in ('pending', 'failed')
277 and available_at <= now()
278 )
279 or (
280 status = 'processing'
281 and locked_at <= now() - ($1::double precision * interval '1 second')
282 )
283 order by available_at asc, created_at asc
284 limit $2
285 for update skip locked
286 )
287 update runtime.function_runs function_run
288 set status = 'processing',
289 locked_at = now(),
290 locked_by = $3,
291 started_at = coalesce(started_at, now()),
292 last_error = null,
293 updated_at = now()
294 from claimed
295 where function_run.id = claimed.id
296 returning
297 function_run.id,
298 function_run.function_name,
299 function_run.input_json,
300 function_run.attempts,
301 function_run.max_attempts,
302 function_run.correlation_id,
303 function_run.actor
304 "#,
305 )
306 .bind(stale_processing_lock_seconds())
307 .bind(batch_size)
308 .bind(&self.worker_id)
309 .fetch_all(&self.pool)
310 .await
311 .map(|rows| {
312 rows.into_iter()
313 .map(TryInto::try_into)
314 .collect::<AppResult<Vec<_>>>()
315 })
316 .map_err(map_runtime_error)??;
317
318 for run in &runs {
319 self.record_function_execution_log(
320 run,
321 ExecutionLogSeverity::Info,
322 "Function run claimed",
323 serde_json::json!({
324 "attempt": run.attempts + 1,
325 "max_attempts": run.max_attempts,
326 "worker_id": self.worker_id,
327 }),
328 )
329 .await;
330 }
331
332 Ok(runs)
333 }
334 .instrument(span)
335 .await
336 }
337
338 pub async fn claim_and_run_batch(&self, batch_size: i64) -> AppResult<usize> {
339 let span = tracing::info_span!(
340 "function_worker_loop",
341 worker_id = %self.worker_id,
342 lenso.execution.kind = "worker_loop",
343 lenso.execution.name = "runtime_worker.claim_and_run_batch",
344 );
345
346 async {
347 let runs = self.claim_batch(batch_size).await?;
348 let count = runs.len();
349
350 for run in runs {
351 self.run_claimed(run).await?;
352 }
353
354 Ok(count)
355 }
356 .instrument(span)
357 .await
358 }
359
360 async fn run_claimed(&self, run: ClaimedFunctionRun) -> AppResult<()> {
361 let span = tracing::info_span!(
362 "function_run",
363 lenso.correlation_id = tracing::field::Empty,
364 lenso.story_id = tracing::field::Empty,
365 lenso.function_run_id = tracing::field::Empty,
366 lenso.execution.kind = tracing::field::Empty,
367 lenso.execution.name = tracing::field::Empty,
368 );
369 record_runtime_span_attributes(
370 &span,
371 &RuntimeSpanAttributes::function(
372 run.correlation_id.clone(),
373 run.id.clone(),
374 run.function_name.clone(),
375 ),
376 );
377
378 async {
379 self.record_function_execution_log(
380 &run,
381 ExecutionLogSeverity::Info,
382 "Function run started",
383 serde_json::json!({
384 "attempt": run.attempts + 1,
385 "max_attempts": run.max_attempts,
386 "worker_id": self.worker_id,
387 }),
388 )
389 .await;
390
391 let Some(definition) = self.registry.get(&run.function_name) else {
392 let error = AppError::new(
393 ErrorCode::Internal,
394 format!("Runtime function {} is not registered", run.function_name),
395 )
396 .retryable();
397 self.mark_failed(&run, &error, RetryPolicy::default().initial_delay)
398 .await?;
399 return Ok(());
400 };
401
402 let attempt = u32::try_from(run.attempts + 1).unwrap_or(u32::MAX);
403 let ctx = ExecutionContext {
404 execution_id: ExecutionId(run.id.clone()),
405 function_name: run.function_name.clone(),
406 attempt,
407 queue: definition.queue.clone(),
408 correlation_id: CorrelationId::new(run.correlation_id.clone()),
409 causation_id: run.causation_id.clone(),
410 actor: run.actor.clone(),
411 tenant_id: None::<TenantId>,
412 trace: run.trace.clone(),
413 deadline: None::<DateTime<Utc>>,
414 };
415
416 let observability = definition.handler.observability();
417 let started_at = Utc::now();
418 let started = Instant::now();
419 let result = definition.handler.call(ctx, run.input_json.clone()).await;
420 let duration_ms = started.elapsed().as_millis().try_into().unwrap_or(i64::MAX);
421 if let Some(observability) = observability {
422 self.record_function_handler_operation_log(
423 &run,
424 observability,
425 started_at,
426 duration_ms,
427 result.as_ref().err(),
428 )
429 .await;
430 }
431
432 match result {
433 Ok(_output) => self.mark_completed(&run).await,
434 Err(error) => {
435 self.mark_failed(&run, &error, definition.retry_policy.initial_delay)
436 .await
437 }
438 }
439 }
440 .instrument(span)
441 .await
442 }
443
444 pub async fn mark_completed(&self, run: &ClaimedFunctionRun) -> AppResult<()> {
445 sqlx::query(
446 r#"
447 update runtime.function_runs
448 set status = 'completed',
449 completed_at = now(),
450 locked_at = null,
451 locked_by = null,
452 last_error = null,
453 updated_at = now()
454 where id = $1
455 "#,
456 )
457 .bind(&run.id)
458 .execute(&self.pool)
459 .await
460 .map(|_| ())
461 .map_err(map_runtime_error)?;
462
463 self.record_function_execution_log(
464 run,
465 ExecutionLogSeverity::Info,
466 "Function run completed",
467 serde_json::json!({
468 "attempt": run.attempts + 1,
469 "max_attempts": run.max_attempts,
470 "worker_id": self.worker_id,
471 }),
472 )
473 .await;
474
475 Ok(())
476 }
477
478 pub async fn mark_failed(
479 &self,
480 run: &ClaimedFunctionRun,
481 error: &AppError,
482 retry_delay: Duration,
483 ) -> AppResult<()> {
484 let next_attempt = run.attempts + 1;
485 let status = if next_attempt >= run.max_attempts {
486 FunctionRunStatus::Dead
487 } else if error.retryable {
488 FunctionRunStatus::Failed
489 } else {
490 FunctionRunStatus::Dead
491 };
492
493 let span = tracing::info_span!(
494 "function_run_fail",
495 lenso.correlation_id = tracing::field::Empty,
496 lenso.story_id = tracing::field::Empty,
497 lenso.function_run_id = tracing::field::Empty,
498 lenso.execution.kind = tracing::field::Empty,
499 lenso.execution.name = tracing::field::Empty,
500 );
501 record_runtime_span_attributes(
502 &span,
503 &RuntimeSpanAttributes::function(
504 run.correlation_id.clone(),
505 run.id.clone(),
506 run.function_name.clone(),
507 ),
508 );
509
510 async {
511 sqlx::query(
512 r#"
513 update runtime.function_runs
514 set status = $2,
515 attempts = attempts + 1,
516 available_at = case
517 when $2 = 'failed' then now() + ($4::double precision * interval '1 second')
518 else available_at
519 end,
520 locked_at = null,
521 locked_by = null,
522 last_error = $3,
523 updated_at = now()
524 where id = $1
525 "#,
526 )
527 .bind(&run.id)
528 .bind(status.as_str())
529 .bind(error.public_message.as_str())
530 .bind(retry_delay_seconds(retry_delay))
531 .execute(&self.pool)
532 .await
533 .map(|_| ())
534 .map_err(map_runtime_error)?;
535
536 self.record_function_execution_log(
537 run,
538 ExecutionLogSeverity::Error,
539 if status == FunctionRunStatus::Dead {
540 "Function run marked dead"
541 } else {
542 "Function run failed"
543 },
544 serde_json::json!({
545 "attempt": next_attempt,
546 "max_attempts": run.max_attempts,
547 "status": status.as_str(),
548 "retryable": error.retryable,
549 "error": error.public_message,
550 "worker_id": self.worker_id,
551 }),
552 )
553 .await;
554
555 Ok(())
556 }
557 .instrument(span)
558 .await
559 }
560
561 async fn record_function_execution_log(
562 &self,
563 run: &ClaimedFunctionRun,
564 severity: ExecutionLogSeverity,
565 body: &'static str,
566 attributes: Value,
567 ) {
568 emit_function_lifecycle_event(run, severity, body, &attributes, Some(&self.worker_id));
569 if let Err(error) = insert_execution_log_projection(
570 &self.pool,
571 function_log_record(run, severity, body, attributes),
572 )
573 .await
574 {
575 tracing::warn!(
576 error = ?error,
577 function_run_id = %run.id,
578 "failed to write function execution log"
579 );
580 }
581 }
582
583 async fn record_function_handler_operation_log(
584 &self,
585 run: &ClaimedFunctionRun,
586 observability: FunctionHandlerObservability,
587 started_at: DateTime<Utc>,
588 duration_ms: i64,
589 error: Option<&AppError>,
590 ) {
591 let body = if error.is_some() {
592 "Function handler operation failed"
593 } else {
594 "Function handler operation completed"
595 };
596 let severity = if error.is_some() {
597 ExecutionLogSeverity::Error
598 } else {
599 ExecutionLogSeverity::Info
600 };
601 let attributes = function_handler_operation_attributes(
602 run,
603 &self.worker_id,
604 observability,
605 duration_ms,
606 error,
607 );
608 emit_function_lifecycle_event(run, severity, body, &attributes, Some(&self.worker_id));
609 if let Err(error) = insert_execution_log_projection(
610 &self.pool,
611 function_log_record(run, severity, body, attributes).with_occurred_at(started_at),
612 )
613 .await
614 {
615 tracing::warn!(
616 error = ?error,
617 function_run_id = %run.id,
618 "failed to write function handler operation log"
619 );
620 }
621 }
622}
623
624#[derive(Debug)]
625struct FunctionLogContext {
626 id: String,
627 function_name: String,
628 correlation_id: String,
629 trace: TraceContext,
630}
631
632trait FunctionLogSource {
633 fn id(&self) -> String;
634 fn function_name(&self) -> String;
635 fn correlation_id(&self) -> String;
636 fn trace(&self) -> TraceContext;
637}
638
639impl FunctionLogSource for FunctionLogContext {
640 fn id(&self) -> String {
641 self.id.clone()
642 }
643
644 fn function_name(&self) -> String {
645 self.function_name.clone()
646 }
647
648 fn correlation_id(&self) -> String {
649 self.correlation_id.clone()
650 }
651
652 fn trace(&self) -> TraceContext {
653 self.trace.clone()
654 }
655}
656
657impl FunctionLogSource for ClaimedFunctionRun {
658 fn id(&self) -> String {
659 self.id.clone()
660 }
661
662 fn function_name(&self) -> String {
663 self.function_name.clone()
664 }
665
666 fn correlation_id(&self) -> String {
667 self.correlation_id.clone()
668 }
669
670 fn trace(&self) -> TraceContext {
671 self.trace.clone()
672 }
673}
674
675fn emit_function_lifecycle_event(
676 run: &impl FunctionLogSource,
677 severity: ExecutionLogSeverity,
678 body: &'static str,
679 attributes: &Value,
680 worker_id: Option<&str>,
681) {
682 match severity {
683 ExecutionLogSeverity::Error => {
684 tracing::error!(
685 function_run_id = %run.id(),
686 function_name = %run.function_name(),
687 correlation_id = %run.correlation_id(),
688 worker_id = worker_id.unwrap_or(""),
689 attributes = %attributes,
690 "{body}"
691 );
692 }
693 _ => {
694 tracing::info!(
695 function_run_id = %run.id(),
696 function_name = %run.function_name(),
697 correlation_id = %run.correlation_id(),
698 worker_id = worker_id.unwrap_or(""),
699 attributes = %attributes,
700 "{body}"
701 );
702 }
703 }
704}
705
706fn function_log_record(
707 run: &impl FunctionLogSource,
708 severity: ExecutionLogSeverity,
709 body: impl Into<String>,
710 attributes: Value,
711) -> ExecutionLogProjectionRecord {
712 ExecutionLogProjectionRecord::from_runtime_attrs(
713 RuntimeSpanAttributes::function(run.correlation_id(), run.id(), run.function_name()),
714 severity,
715 body,
716 )
717 .with_attributes(attributes)
718 .with_trace(run.trace())
719}
720
721#[derive(Debug, Clone)]
722struct ExecutionLogProjectionRecord {
723 correlation_id: String,
724 execution_id: String,
725 execution_type: String,
726 execution_name: String,
727 severity: ExecutionLogSeverity,
728 body: String,
729 attributes: Value,
730 trace: TraceContext,
731 service_name: String,
732 occurred_at: Option<DateTime<Utc>>,
733}
734
735impl ExecutionLogProjectionRecord {
736 fn from_runtime_attrs(
737 attrs: RuntimeSpanAttributes,
738 severity: ExecutionLogSeverity,
739 body: impl Into<String>,
740 ) -> Self {
741 let execution_id = attrs
742 .function_run_id
743 .clone()
744 .or(attrs.outbox_event_id)
745 .unwrap_or_else(|| attrs.story_id.clone());
746
747 Self {
748 correlation_id: attrs.correlation_id,
749 execution_id,
750 execution_type: attrs.execution_kind,
751 execution_name: attrs.execution_name,
752 severity,
753 body: body.into(),
754 attributes: Value::Object(Default::default()),
755 trace: TraceContext::default(),
756 service_name: "lenso".to_owned(),
757 occurred_at: None,
758 }
759 }
760
761 fn with_attributes(mut self, attributes: Value) -> Self {
762 self.attributes = attributes;
763 self
764 }
765
766 fn with_trace(mut self, trace: TraceContext) -> Self {
767 self.trace = trace;
768 self
769 }
770
771 fn with_occurred_at(mut self, occurred_at: DateTime<Utc>) -> Self {
772 self.occurred_at = Some(occurred_at);
773 self
774 }
775}
776
777async fn insert_execution_log_projection(
778 pool: &DbPool,
779 record: ExecutionLogProjectionRecord,
780) -> AppResult<String> {
781 let id = format!("elog_{}", Uuid::now_v7());
782 let occurred_at = record.occurred_at.unwrap_or_else(Utc::now);
783
784 sqlx::query(
785 r#"
786 insert into platform.execution_logs (
787 id,
788 correlation_id,
789 story_id,
790 execution_id,
791 execution_type,
792 execution_name,
793 occurred_at,
794 severity,
795 body,
796 attributes,
797 trace_id,
798 span_id,
799 service_name,
800 redacted_fields
801 )
802 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
803 "#,
804 )
805 .bind(&id)
806 .bind(&record.correlation_id)
807 .bind(&record.correlation_id)
808 .bind(&record.execution_id)
809 .bind(&record.execution_type)
810 .bind(&record.execution_name)
811 .bind(occurred_at)
812 .bind(record.severity.as_str())
813 .bind(&record.body)
814 .bind(normalize_log_attributes(record.attributes))
815 .bind(&record.trace.trace_id)
816 .bind(&record.trace.span_id)
817 .bind(&record.service_name)
818 .bind(Vec::<String>::new())
819 .execute(pool)
820 .await
821 .map_err(map_runtime_error)?;
822
823 Ok(id)
824}
825
826fn function_handler_operation_attributes(
827 run: &ClaimedFunctionRun,
828 worker_id: &str,
829 observability: FunctionHandlerObservability,
830 duration_ms: i64,
831 error: Option<&AppError>,
832) -> Value {
833 let mut attributes = match observability.attributes {
834 Value::Object(attributes) => attributes,
835 other => serde_json::Map::from_iter([("value".to_owned(), other)]),
836 };
837 attributes.insert("source".to_owned(), Value::String(observability.source));
838 attributes.insert("attempt".to_owned(), serde_json::json!(run.attempts + 1));
839 attributes.insert(
840 "max_attempts".to_owned(),
841 serde_json::json!(run.max_attempts),
842 );
843 attributes.insert("duration_ms".to_owned(), serde_json::json!(duration_ms));
844 attributes.insert("success".to_owned(), serde_json::json!(error.is_none()));
845 attributes.insert("worker_id".to_owned(), serde_json::json!(worker_id));
846 attributes.insert(
847 "function_name".to_owned(),
848 serde_json::json!(run.function_name),
849 );
850 attributes.insert("request_id".to_owned(), serde_json::json!(run.id));
851 attributes.insert("trace_id".to_owned(), serde_json::json!(run.trace.trace_id));
852 attributes.insert("span_id".to_owned(), serde_json::json!(run.trace.span_id));
853
854 if let Some(error) = error {
855 attributes.insert(
856 "error_code".to_owned(),
857 serde_json::json!(error.code.as_str()),
858 );
859 attributes.insert("error".to_owned(), serde_json::json!(error.public_message));
860 attributes.insert("retryable".to_owned(), serde_json::json!(error.retryable));
861 attributes.insert("error_details".to_owned(), serde_json::json!(error.details));
862 }
863
864 Value::Object(attributes)
865}
866
867fn normalize_log_attributes(attributes: Value) -> Value {
868 match attributes {
869 Value::Object(_) => attributes,
870 other => serde_json::json!({ "value": other }),
871 }
872}
873
874fn stale_processing_lock_seconds() -> f64 {
875 STALE_PROCESSING_LOCK_SECONDS as f64
876}
877
878fn retry_delay_seconds(delay: Duration) -> f64 {
879 delay.as_secs_f64()
880}
881
882type FunctionRunRow = (String, String, Value, i32, i32, String, Value);
883
884impl TryFrom<FunctionRunRow> for ClaimedFunctionRun {
885 type Error = AppError;
886
887 fn try_from(row: FunctionRunRow) -> Result<Self, Self::Error> {
888 let (id, function_name, input_json, attempts, max_attempts, correlation_id, actor) = row;
889 let runtime_context = input_json.get("_lenso_runtime");
890 let trace = runtime_context
891 .map(trace_context_from_headers)
892 .unwrap_or_default();
893 let causation_id = runtime_context
894 .and_then(|context| context.get("causation_id"))
895 .and_then(Value::as_str)
896 .map(ToOwned::to_owned);
897 Ok(Self {
898 id,
899 function_name,
900 input_json,
901 attempts,
902 max_attempts,
903 correlation_id,
904 actor: serde_json::from_value(actor).map_err(map_serde_error)?,
905 trace,
906 causation_id,
907 })
908 }
909}
910
911fn attach_runtime_context_to_input(
912 input_json: &mut Value,
913 correlation_id: &CorrelationId,
914 trace: &TraceContext,
915 causation_id: Option<&str>,
916) {
917 let mut runtime_context = trace_headers(trace, correlation_id);
918 if let Some(causation_id) = causation_id {
919 runtime_context["causation_id"] = Value::String(causation_id.to_owned());
920 }
921
922 match input_json {
923 Value::Object(object) => {
924 object.insert("_lenso_runtime".to_owned(), runtime_context);
925 }
926 other => {
927 *other = serde_json::json!({
928 "payload": other.clone(),
929 "_lenso_runtime": runtime_context,
930 });
931 }
932 }
933}
934
935fn map_runtime_error(source: sqlx::Error) -> AppError {
936 AppError::new(ErrorCode::Internal, "Runtime operation failed").with_source(source)
937}
938
939fn map_serde_error(source: serde_json::Error) -> AppError {
940 AppError::new(ErrorCode::Internal, "Runtime payload serialization failed").with_source(source)
941}