rustvello 0.8.0

Distributed task queue and workflow runtime for Rust and Python: typed tasks, retries, priorities, triggers and pluggable backends
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

//! Recovery and runner maintenance use cases.

use rustvello_core::error::{RustvelloError, RustvelloResult};
use rustvello_proto::config::AppConfig;
use rustvello_proto::identifiers::{InvocationId, RunnerId};
use rustvello_proto::status::InvocationStatus;

use crate::task_catalog::TaskCatalog;

use super::Orchestrator;

impl Orchestrator {
    /// Recover stale pending/running invocations and republish them once.
    pub(crate) async fn recover_stale_invocations(
        &self,
        app_config: &AppConfig,
        task_catalog: &TaskCatalog,
        runner_id: &RunnerId,
    ) -> RustvelloResult<u32> {
        let mut recovered = 0;
        let stale_pending = self
            .backends
            .invocation_control
            .get_stale_pending_invocations(app_config.max_pending_seconds)
            .await?;
        for invocation_id in stale_pending {
            if self
                .recover_one(
                    app_config,
                    task_catalog,
                    runner_id,
                    &invocation_id,
                    InvocationStatus::PendingRecovery,
                )
                .await?
            {
                recovered += 1;
            }
        }

        let stale_running = self
            .backends
            .invocation_control
            .get_stale_running_invocations(app_config.runner_dead_after_seconds)
            .await?;
        for invocation_id in stale_running {
            if self
                .recover_one(
                    app_config,
                    task_catalog,
                    runner_id,
                    &invocation_id,
                    InvocationStatus::RunningRecovery,
                )
                .await?
            {
                recovered += 1;
            }
        }

        Ok(recovered)
    }

    async fn recover_one(
        &self,
        app_config: &AppConfig,
        task_catalog: &TaskCatalog,
        runner_id: &RunnerId,
        invocation_id: &InvocationId,
        recovery_status: InvocationStatus,
    ) -> RustvelloResult<bool> {
        if let Some(publication) = self.publication()? {
            let invocation = self
                .backends
                .state_backend
                .get_invocation(invocation_id)
                .await?;
            let (queue, priority) = task_catalog
                .routing_for(app_config, &invocation.task_id)
                .ok_or_else(|| RustvelloError::TaskNotRegistered {
                    task_id: invocation.task_id.clone(),
                })?;
            let stale_after_seconds = if recovery_status == InvocationStatus::PendingRecovery {
                app_config.max_pending_seconds
            } else {
                app_config.runner_dead_after_seconds
            };
            let recovered = publication
                .change(
                    invocation_id,
                    runner_id,
                    rustvello_core::publication::PublicationChange::Recover {
                        status: recovery_status,
                        stale_after_seconds,
                        route: rustvello_core::publication::PublicationRoute { queue, priority },
                    },
                    false,
                )
                .await?
                .is_some();
            if recovered {
                let arguments = self.get_invocation_arguments(invocation_id).await;
                for status in [recovery_status, InvocationStatus::Rerouted] {
                    self.report_published_status(
                        invocation_id,
                        runner_id,
                        status,
                        &invocation.task_id,
                        arguments.clone(),
                    )
                    .await?;
                }
            }
            return Ok(recovered);
        }
        match self
            .set_invocation_status(invocation_id, recovery_status, runner_id)
            .await
        {
            Ok(_) => {}
            Err(RustvelloError::InvalidStatusTransition { .. }) => return Ok(false),
            Err(error) => return Err(error),
        }

        let invocation = self
            .backends
            .state_backend
            .get_invocation(invocation_id)
            .await?;
        let (queue, priority) = task_catalog
            .routing_for(app_config, &invocation.task_id)
            .ok_or_else(|| RustvelloError::TaskNotRegistered {
                task_id: invocation.task_id.clone(),
            })?;

        self.set_invocation_status(invocation_id, InvocationStatus::Rerouted, runner_id)
            .await?;
        self.backends
            .broker
            .route_invocation_with_options(
                invocation_id,
                Some(&invocation.task_id),
                &queue,
                priority,
            )
            .await?;
        Ok(true)
    }
}