1use super::*;
2
3#[cfg(test)]
4#[path = "recovery_tests.rs"]
5mod tests;
6
7pub const PROJECT_LOST_ORIGINAL_CODE: &str = "projectLost";
9
10pub const ACTIVE_PROJECTS_RECOVERED_EVENT: &str = "activeProjectsRecovered";
12
13pub const COMPLETED_PROJECTS_RECOVERED_EVENT: &str = "completedProjectsRecovered";
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq)]
18pub struct ResolveMissingOptions {
19 pub attempts: usize,
21 pub retry_delay: Duration,
23}
24
25impl Default for ResolveMissingOptions {
26 fn default() -> Self {
27 Self {
28 attempts: MISSING_PROJECT_ATTEMPTS,
29 retry_delay: MISSING_PROJECT_RETRY,
30 }
31 }
32}
33
34#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
36#[serde(tag = "state", rename_all = "lowercase")]
37pub enum ProjectResolution {
38 Finished { project: Value },
40 Active,
42 Lost,
45 Unknown { error: String },
47}
48
49impl ProjectResolution {
50 #[must_use]
52 pub const fn state(&self) -> &'static str {
53 match self {
54 Self::Finished { .. } => "finished",
55 Self::Active => "active",
56 Self::Lost => "lost",
57 Self::Unknown { .. } => "unknown",
58 }
59 }
60}
61
62pub(super) fn project_lost_payload() -> Value {
63 json!({
64 "code": 0,
65 "originalCode": PROJECT_LOST_ORIGINAL_CODE,
66 "message": "The server has no record of this generation. It may have been interrupted by a restart — please try again."
67 })
68}
69
70#[must_use]
72pub fn is_project_lost_error(error: &Error) -> bool {
73 matches!(
74 error,
75 Error::Project(error)
76 if error.payload.get("originalCode").and_then(Value::as_str)
77 == Some(PROJECT_LOST_ORIGINAL_CODE)
78 )
79}
80
81#[must_use]
83pub fn is_project_lost_payload(payload: &Value) -> bool {
84 payload.get("originalCode").and_then(Value::as_str) == Some(PROJECT_LOST_ORIGINAL_CODE)
85}
86
87pub(super) fn recovered_params(raw: &Value) -> Value {
88 let request = raw
89 .get("clientRequestData")
90 .and_then(Value::as_str)
91 .and_then(|value| crate::utils::b64_json_decode(value).ok())
92 .unwrap_or_else(|| json!({}));
93 let keyframe = request
94 .pointer("/keyFrames/0")
95 .filter(|value| value.is_object())
96 .cloned()
97 .unwrap_or_else(|| json!({}));
98 let model = raw.get("model").cloned().unwrap_or_else(|| json!({}));
99 let model_type = raw
100 .get("modelType")
101 .or_else(|| model.get("type"))
102 .and_then(Value::as_str)
103 .unwrap_or("image");
104 let media_type = match model_type {
105 "video" => "video",
106 "audio" | "music" => "audio",
107 _ => "image",
108 };
109 let count = raw
110 .get("imageCount")
111 .and_then(Value::as_u64)
112 .or_else(|| request.get("numberOfImages").and_then(Value::as_u64))
113 .unwrap_or(1);
114 let mut params = json!({
115 "type": media_type,
116 "modelId": keyframe.get("modelID").or_else(|| model.get("id")).cloned().unwrap_or(json!("")),
117 "positivePrompt": keyframe.get("positivePrompt").cloned().unwrap_or(json!("")),
118 "numberOfMedia": count,
119 });
120 for (source, target) in [
121 ("negativePrompt", "negativePrompt"),
122 ("stylePrompt", "stylePrompt"),
123 ("guidanceScale", "guidance"),
124 ("seed", "seed"),
125 ("loras", "loras"),
126 ("loraStrengths", "loraStrengths"),
127 ("sam3Prompt", "sam3Prompt"),
128 ("worldGenerationReceipt", "worldGenerationReceipt"),
129 ] {
130 if let Some(value) = keyframe.get(source).filter(|value| !value.is_null()) {
131 params[target] = value.clone();
132 }
133 }
134 if let Some(steps) = raw.get("stepCount").or_else(|| keyframe.get("steps")) {
135 params["steps"] = steps.clone();
136 }
137 for field in ["network", "tokenType", "billingMode", "appSource"] {
138 if let Some(value) = raw.get(field).or_else(|| request.get(field)) {
139 params[field] = value.clone();
140 }
141 }
142 if let Some(value) = request.get("outputFormat") {
143 params["outputFormat"] = value.clone();
144 }
145 wire::normalize_utility_params(params.as_object_mut().expect("recovered params object"));
146 params
147}
148
149pub(super) fn replay_recovered(project: &Project, raw: &Value, completed: bool) {
150 let mut jobs = Vec::new();
151 if !completed {
152 jobs.extend(
153 raw.get("workerJobs")
154 .and_then(Value::as_array)
155 .cloned()
156 .unwrap_or_default(),
157 );
158 }
159 jobs.extend(
160 raw.get("completedWorkerJobs")
161 .and_then(Value::as_array)
162 .cloned()
163 .unwrap_or_default(),
164 );
165 for job_raw in jobs {
166 let Some(id) = job_raw
167 .get("imgID")
168 .or_else(|| job_raw.get("id"))
169 .and_then(Value::as_str)
170 else {
171 continue;
172 };
173 let Some(job) =
174 project.job_for_attempt(id, job_raw.get("jobIndex").and_then(Value::as_u64))
175 else {
176 continue;
177 };
178 let raw_status = job_raw.get("status").and_then(Value::as_str).unwrap_or("");
179 let status = match raw_status {
180 "assigned" | "initiatingModel" => JobStatus::Initiating,
181 "jobStarted" | "jobProgress" => JobStatus::Processing,
182 "jobCompleted" => JobStatus::Completed,
183 "jobError" => JobStatus::Failed,
184 _ => JobStatus::Pending,
185 };
186 job.update(
187 |state| {
188 if state.status.is_finished() && !status.is_finished() {
189 return;
190 }
191 if !state.status.is_finished() {
192 state.status = status;
193 }
194 state.step = number(job_raw.get("performedSteps")).unwrap_or(state.step);
195 state.seed = job_raw
196 .get("seedUsed")
197 .and_then(Value::as_i64)
198 .or(state.seed);
199 state.worker_name = job_raw
200 .pointer("/worker/username")
201 .or_else(|| job_raw.pointer("/worker/name"))
202 .and_then(Value::as_str)
203 .map(ToOwned::to_owned)
204 .or_else(|| state.worker_name.clone());
205 state.result_url = raw_result_url(&job_raw).or_else(|| state.result_url.clone());
206 super::events::copy_export_metadata(state, &job_raw);
207 if let Some(index) = job_raw.get("jobIndex") {
208 state.extra.insert("jobIndex".into(), index.clone());
209 }
210 if let Some(provenance) = JobProvenance::from_result(&job_raw) {
211 state.provenance = Some(provenance);
212 }
213 if let Some(preparation) = job_raw.get("preparation") {
214 state
215 .extra
216 .insert("preparation".into(), preparation.clone());
217 }
218 state.is_nsfw = job_raw
219 .get("triggeredNSFWFilter")
220 .and_then(Value::as_bool)
221 .unwrap_or(false);
222 state.nsfw_detected = job_raw
223 .get("nsfwDetected")
224 .and_then(Value::as_bool)
225 .unwrap_or(false);
226 state.nsfw_sources = job_raw
227 .get("nsfwSources")
228 .and_then(Value::as_array)
229 .into_iter()
230 .flatten()
231 .filter_map(Value::as_str)
232 .map(ToOwned::to_owned)
233 .collect();
234 },
235 &["status", "step", "resultUrl", "provenance", "preparation"],
236 );
237 }
238 let status = match raw.get("status").and_then(Value::as_str) {
239 Some("completed") => Some(ProjectStatus::Completed),
240 Some("errored" | "failed") => Some(ProjectStatus::Failed),
241 Some("cancelled" | "canceled") => Some(ProjectStatus::Canceled),
242 Some("queued" | "active") => Some(ProjectStatus::Queued),
243 Some("processing") => Some(ProjectStatus::Processing),
244 Some("pending") => Some(ProjectStatus::Pending),
245 _ if completed => Some(ProjectStatus::Completed),
246 _ => None,
247 };
248 if let Some(status) = status {
249 if raw.get("status").and_then(Value::as_str) == Some("queued") {
250 project.suspend_processing_deadlines();
251 }
252 let snapshot = project.snapshot();
255 let terminal_error = if matches!(status, ProjectStatus::Failed | ProjectStatus::Canceled)
256 && (!snapshot.status.is_finished() || snapshot.status == status)
257 {
258 Some(
259 snapshot
260 .error
261 .unwrap_or_else(|| recovered_terminal_error(raw, status)),
262 )
263 } else {
264 None
265 };
266 if let Some(error) = &terminal_error {
267 for job in project.jobs() {
268 job.update(
269 |state| {
270 if !state.status.is_finished() {
271 state.status = if status == ProjectStatus::Canceled {
272 JobStatus::Canceled
273 } else {
274 JobStatus::Failed
275 };
276 state.error = Some(error.clone());
277 }
278 },
279 &["status", "error"],
280 );
281 }
282 }
283 project.update(
284 |state| {
285 if !state.status.is_finished() || state.status == status {
286 state.status = status;
287 if terminal_error.is_some() {
288 state.error = terminal_error;
289 }
290 }
291 },
292 &["status", "jobs", "error"],
293 );
294 }
295}
296
297fn recovered_terminal_error(raw: &Value, status: ProjectStatus) -> Value {
298 let reason = raw
299 .get("reason")
300 .and_then(Value::as_str)
301 .unwrap_or_default()
302 .trim();
303 let code = reason
304 .bytes()
305 .all(|b| b.is_ascii_digit())
306 .then(|| reason.parse::<u64>().ok())
307 .flatten()
308 .filter(|code| *code <= 9_007_199_254_740_991)
309 .unwrap_or(0);
310 let message = if reason.is_empty() {
311 if status == ProjectStatus::Canceled {
312 "Project canceled"
313 } else {
314 "Project failed"
315 }
316 } else {
317 reason
318 };
319 json!({"code":code, "message":message})
320}
321
322pub(super) fn is_llm_recovery(raw: &Value) -> bool {
323 raw.get("jobType").and_then(Value::as_str) == Some("llm")
324 || raw.pointer("/model/type").and_then(Value::as_str) == Some("llm")
325}
326
327pub(super) fn raw_result_url(data: &Value) -> Option<String> {
328 [
329 "resultUrl",
330 "imageUrl",
331 "imageFile",
332 "videoUrl",
333 "videoFile",
334 ]
335 .iter()
336 .find_map(|field| {
337 data.get(field)
338 .and_then(Value::as_str)
339 .filter(|url| !url.is_empty())
340 .map(ToOwned::to_owned)
341 })
342}