Skip to main content

sogni_client/projects/
recovery.rs

1use super::*;
2
3#[cfg(test)]
4#[path = "recovery_tests.rs"]
5mod tests;
6
7/// Stable marker stored in `originalCode` when reconciliation proves a project was lost.
8pub const PROJECT_LOST_ORIGINAL_CODE: &str = "projectLost";
9
10/// Event emitted with newly rehydrated in-flight recovery records.
11pub const ACTIVE_PROJECTS_RECOVERED_EVENT: &str = "activeProjectsRecovered";
12
13/// Event emitted with newly rehydrated completed recovery records.
14pub const COMPLETED_PROJECTS_RECOVERED_EVENT: &str = "completedProjectsRecovered";
15
16/// Retry policy for [`ProjectsApi::resolve_missing`].
17#[derive(Clone, Copy, Debug, PartialEq, Eq)]
18pub struct ResolveMissingOptions {
19    /// Number of REST lookups before consulting the socket's active-project list.
20    pub attempts: usize,
21    /// Delay between REST lookups after the first attempt.
22    pub retry_delay: Duration,
23}
24
25impl Default for ResolveMissingOptions {
26    fn default() -> Self {
27        Self {
28            attempts: MISSING_PROJECT_ATTEMPTS,
29            retry_delay: MISSING_PROJECT_RETRY,
30        }
31    }
32}
33
34/// Outcome of looking up a project omitted from a recovery snapshot.
35#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
36#[serde(tag = "state", rename_all = "lowercase")]
37pub enum ProjectResolution {
38    /// The durable REST record confirms a terminal project (success, failure or cancellation).
39    Finished { project: Value },
40    /// The v2 REST status or socket registry still owns the in-flight project.
41    Active,
42    /// Neither the durable REST API nor the socket currently has a record.
43    /// This does not establish that a historical project never existed.
44    Lost,
45    /// A non-404 error prevented a reliable verdict.
46    Unknown { error: String },
47}
48
49impl ProjectResolution {
50    /// Stable lower-case state name matching the JavaScript and Python clients.
51    #[must_use]
52    pub const fn state(&self) -> &'static str {
53        match self {
54            Self::Finished { .. } => "finished",
55            Self::Active => "active",
56            Self::Lost => "lost",
57            Self::Unknown { .. } => "unknown",
58        }
59    }
60}
61
62pub(super) fn project_lost_payload() -> Value {
63    json!({
64        "code": 0,
65        "originalCode": PROJECT_LOST_ORIGINAL_CODE,
66        "message": "The server has no record of this generation. It may have been interrupted by a restart — please try again."
67    })
68}
69
70/// Returns `true` for the failure assigned when reconciliation cannot find a project.
71#[must_use]
72pub fn is_project_lost_error(error: &Error) -> bool {
73    matches!(
74        error,
75        Error::Project(error)
76            if error.payload.get("originalCode").and_then(Value::as_str)
77                == Some(PROJECT_LOST_ORIGINAL_CODE)
78    )
79}
80
81/// Returns `true` when a raw project-error payload carries the lost-project marker.
82#[must_use]
83pub fn is_project_lost_payload(payload: &Value) -> bool {
84    payload.get("originalCode").and_then(Value::as_str) == Some(PROJECT_LOST_ORIGINAL_CODE)
85}
86
87pub(super) fn recovered_params(raw: &Value) -> Value {
88    let request = raw
89        .get("clientRequestData")
90        .and_then(Value::as_str)
91        .and_then(|value| crate::utils::b64_json_decode(value).ok())
92        .unwrap_or_else(|| json!({}));
93    let keyframe = request
94        .pointer("/keyFrames/0")
95        .filter(|value| value.is_object())
96        .cloned()
97        .unwrap_or_else(|| json!({}));
98    let model = raw.get("model").cloned().unwrap_or_else(|| json!({}));
99    let model_type = raw
100        .get("modelType")
101        .or_else(|| model.get("type"))
102        .and_then(Value::as_str)
103        .unwrap_or("image");
104    let media_type = match model_type {
105        "video" => "video",
106        "audio" | "music" => "audio",
107        _ => "image",
108    };
109    let count = raw
110        .get("imageCount")
111        .and_then(Value::as_u64)
112        .or_else(|| request.get("numberOfImages").and_then(Value::as_u64))
113        .unwrap_or(1);
114    let mut params = json!({
115        "type": media_type,
116        "modelId": keyframe.get("modelID").or_else(|| model.get("id")).cloned().unwrap_or(json!("")),
117        "positivePrompt": keyframe.get("positivePrompt").cloned().unwrap_or(json!("")),
118        "numberOfMedia": count,
119    });
120    for (source, target) in [
121        ("negativePrompt", "negativePrompt"),
122        ("stylePrompt", "stylePrompt"),
123        ("guidanceScale", "guidance"),
124        ("seed", "seed"),
125        ("loras", "loras"),
126        ("loraStrengths", "loraStrengths"),
127        ("sam3Prompt", "sam3Prompt"),
128        ("worldGenerationReceipt", "worldGenerationReceipt"),
129    ] {
130        if let Some(value) = keyframe.get(source).filter(|value| !value.is_null()) {
131            params[target] = value.clone();
132        }
133    }
134    if let Some(steps) = raw.get("stepCount").or_else(|| keyframe.get("steps")) {
135        params["steps"] = steps.clone();
136    }
137    for field in ["network", "tokenType", "billingMode", "appSource"] {
138        if let Some(value) = raw.get(field).or_else(|| request.get(field)) {
139            params[field] = value.clone();
140        }
141    }
142    if let Some(value) = request.get("outputFormat") {
143        params["outputFormat"] = value.clone();
144    }
145    wire::normalize_utility_params(params.as_object_mut().expect("recovered params object"));
146    params
147}
148
149pub(super) fn replay_recovered(project: &Project, raw: &Value, completed: bool) {
150    let mut jobs = Vec::new();
151    if !completed {
152        jobs.extend(
153            raw.get("workerJobs")
154                .and_then(Value::as_array)
155                .cloned()
156                .unwrap_or_default(),
157        );
158    }
159    jobs.extend(
160        raw.get("completedWorkerJobs")
161            .and_then(Value::as_array)
162            .cloned()
163            .unwrap_or_default(),
164    );
165    for job_raw in jobs {
166        let Some(id) = job_raw
167            .get("imgID")
168            .or_else(|| job_raw.get("id"))
169            .and_then(Value::as_str)
170        else {
171            continue;
172        };
173        let Some(job) =
174            project.job_for_attempt(id, job_raw.get("jobIndex").and_then(Value::as_u64))
175        else {
176            continue;
177        };
178        let raw_status = job_raw.get("status").and_then(Value::as_str).unwrap_or("");
179        let status = match raw_status {
180            "assigned" | "initiatingModel" => JobStatus::Initiating,
181            "jobStarted" | "jobProgress" => JobStatus::Processing,
182            "jobCompleted" => JobStatus::Completed,
183            "jobError" => JobStatus::Failed,
184            _ => JobStatus::Pending,
185        };
186        job.update(
187            |state| {
188                if state.status.is_finished() && !status.is_finished() {
189                    return;
190                }
191                if !state.status.is_finished() {
192                    state.status = status;
193                }
194                state.step = number(job_raw.get("performedSteps")).unwrap_or(state.step);
195                state.seed = job_raw
196                    .get("seedUsed")
197                    .and_then(Value::as_i64)
198                    .or(state.seed);
199                state.worker_name = job_raw
200                    .pointer("/worker/username")
201                    .or_else(|| job_raw.pointer("/worker/name"))
202                    .and_then(Value::as_str)
203                    .map(ToOwned::to_owned)
204                    .or_else(|| state.worker_name.clone());
205                state.result_url = raw_result_url(&job_raw).or_else(|| state.result_url.clone());
206                super::events::copy_export_metadata(state, &job_raw);
207                if let Some(index) = job_raw.get("jobIndex") {
208                    state.extra.insert("jobIndex".into(), index.clone());
209                }
210                if let Some(provenance) = JobProvenance::from_result(&job_raw) {
211                    state.provenance = Some(provenance);
212                }
213                if let Some(preparation) = job_raw.get("preparation") {
214                    state
215                        .extra
216                        .insert("preparation".into(), preparation.clone());
217                }
218                state.is_nsfw = job_raw
219                    .get("triggeredNSFWFilter")
220                    .and_then(Value::as_bool)
221                    .unwrap_or(false);
222                state.nsfw_detected = job_raw
223                    .get("nsfwDetected")
224                    .and_then(Value::as_bool)
225                    .unwrap_or(false);
226                state.nsfw_sources = job_raw
227                    .get("nsfwSources")
228                    .and_then(Value::as_array)
229                    .into_iter()
230                    .flatten()
231                    .filter_map(Value::as_str)
232                    .map(ToOwned::to_owned)
233                    .collect();
234            },
235            &["status", "step", "resultUrl", "provenance", "preparation"],
236        );
237    }
238    let status = match raw.get("status").and_then(Value::as_str) {
239        Some("completed") => Some(ProjectStatus::Completed),
240        Some("errored" | "failed") => Some(ProjectStatus::Failed),
241        Some("cancelled" | "canceled") => Some(ProjectStatus::Canceled),
242        Some("queued" | "active") => Some(ProjectStatus::Queued),
243        Some("processing") => Some(ProjectStatus::Processing),
244        Some("pending") => Some(ProjectStatus::Pending),
245        _ if completed => Some(ProjectStatus::Completed),
246        _ => None,
247    };
248    if let Some(status) = status {
249        if raw.get("status").and_then(Value::as_str) == Some("queued") {
250            project.suspend_processing_deadlines();
251        }
252        // A compact terminal record may omit unfinished children entirely.
253        // Preserve an existing server error and settle every remaining child.
254        let snapshot = project.snapshot();
255        let terminal_error = if matches!(status, ProjectStatus::Failed | ProjectStatus::Canceled)
256            && (!snapshot.status.is_finished() || snapshot.status == status)
257        {
258            Some(
259                snapshot
260                    .error
261                    .unwrap_or_else(|| recovered_terminal_error(raw, status)),
262            )
263        } else {
264            None
265        };
266        if let Some(error) = &terminal_error {
267            for job in project.jobs() {
268                job.update(
269                    |state| {
270                        if !state.status.is_finished() {
271                            state.status = if status == ProjectStatus::Canceled {
272                                JobStatus::Canceled
273                            } else {
274                                JobStatus::Failed
275                            };
276                            state.error = Some(error.clone());
277                        }
278                    },
279                    &["status", "error"],
280                );
281            }
282        }
283        project.update(
284            |state| {
285                if !state.status.is_finished() || state.status == status {
286                    state.status = status;
287                    if terminal_error.is_some() {
288                        state.error = terminal_error;
289                    }
290                }
291            },
292            &["status", "jobs", "error"],
293        );
294    }
295}
296
297fn recovered_terminal_error(raw: &Value, status: ProjectStatus) -> Value {
298    let reason = raw
299        .get("reason")
300        .and_then(Value::as_str)
301        .unwrap_or_default()
302        .trim();
303    let code = reason
304        .bytes()
305        .all(|b| b.is_ascii_digit())
306        .then(|| reason.parse::<u64>().ok())
307        .flatten()
308        .filter(|code| *code <= 9_007_199_254_740_991)
309        .unwrap_or(0);
310    let message = if reason.is_empty() {
311        if status == ProjectStatus::Canceled {
312            "Project canceled"
313        } else {
314            "Project failed"
315        }
316    } else {
317        reason
318    };
319    json!({"code":code, "message":message})
320}
321
322pub(super) fn is_llm_recovery(raw: &Value) -> bool {
323    raw.get("jobType").and_then(Value::as_str) == Some("llm")
324        || raw.pointer("/model/type").and_then(Value::as_str) == Some("llm")
325}
326
327pub(super) fn raw_result_url(data: &Value) -> Option<String> {
328    [
329        "resultUrl",
330        "imageUrl",
331        "imageFile",
332        "videoUrl",
333        "videoFile",
334    ]
335    .iter()
336    .find_map(|field| {
337        data.get(field)
338            .and_then(Value::as_str)
339            .filter(|url| !url.is_empty())
340            .map(ToOwned::to_owned)
341    })
342}